diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c71f1c4a2..64de3f752 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,7 +16,7 @@ jobs: # every job passed. See docs/engineering/skills/github-prs.md. name: CI complete if: ${{ always() }} - needs: [test, lint, docker-build, test-integration] + needs: [test, lint, quality-guards, docker-build, test-integration] runs-on: ubuntu-latest permissions: contents: read @@ -107,6 +107,29 @@ jobs: fi done + quality-guards: + name: Quality guards + # Full CI runs only on the lowest unmerged pull request in a stack. + if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + # The rules registry guard compares baselines with the base branch. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Test the rules registry tool + run: python -m unittest discover -s scripts/rules/tests -t . + + - name: Run quality guards + run: python scripts/run_quality_guards.py + docker-build: name: Docker build needs: [test, lint] # Only run if tests and linting pass diff --git a/AGENTS.md b/AGENTS.md index 0f75d5d42..52c8bc159 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,10 @@ Before changing correlation identifiers, telemetry transport, spans, stage names, logging, or observability failure handling, read `docs/engineering/skills/observability.md`. +When adding, changing, or removing a rule, a guard test, a checker +configuration, or a CI job that enforces one, read +`docs/engineering/skills/rules-registry.md`. + ## Development - Use Python 3.13 and `uv`. @@ -59,6 +63,77 @@ same-repository draft PR. - The simulation service lives in `projects/policyengine-simulation-executor`. - API integration tests live in `projects/policyengine-apis-integ`. -- PR CI runs simulation unit tests, Ruff format checks, Docker build, and local - integration tests. +- PR CI runs simulation unit tests, Ruff format checks, quality guards + (including the rules registry), Docker build, and local integration tests. - There is currently no repository-wide changelog fragment requirement. + +## Rules + + + +Every rule for this repository lives in `rules/registry.toml` and is rendered, +with its enforcement and gaps, in `docs/generated/rules.md`. Blocking rules +fail CI. Doc-only rules are not checked automatically, so follow them yourself. +To add or change a rule, read `docs/engineering/skills/rules-registry.md`. + +### Service and package boundaries + +- `sim.boundaries.gateway-executor-decoupled`: The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. (doc-only, not checked: follow it yourself) +- `sim.boundaries.unit-tests-need-no-services`: Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. (doc-only, not checked: follow it yourself) + +### Stage 12 persistence + +- `sim.stage12.deployment-scope`: Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.stage12.deployment-uses-api-schema`: Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.stage12.no-migration-chain`: This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) +- `sim.stage12.no-production-user-associations`: Stage 12 runtime modules never create production user associations (user_simulations or user_reports). (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) +- `sim.stage12.no-schema-ddl`: Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py`) +- `sim.stage12.persistence-typed-dml-only`: The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. (blocking: pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py`) +- `sim.stage12.production-lock-has-db-runtime`: The production lock file includes the direct Stage 12 database runtime dependencies. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) + +### Deployment pipeline + +- `sim.deploy.container-context-minimal`: Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.docker-images-build`: The simulation entry and executor Docker images build. (blocking: CI job `docker-build`) +- `sim.deploy.gcloud-without-terraform`: Deployment uses the gcloud-based workflow, not Terraform. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. (doc-only, not checked: follow it yourself) +- `sim.deploy.modal-image-smoke`: App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes. (blocking: CI job `image-smoke`) +- `sim.deploy.promotion-order-explicit`: Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.revision-tags-valid`: Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.shell-scripts-valid`: Deployment shell scripts have valid shell syntax. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.smoke-tests-strict`: Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.traffic-changes-exact`: Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.deploy.workflow-shell-safety`: Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) + +### API contracts and integration + +- `sim.contracts.clients-publish-deployed-commit`: Client publication checks out the commit that was actually deployed. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.contracts.full-integration-beta-only`: The complete integration suite runs against beta deployments only, never production. (blocking: pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py`) +- `sim.contracts.gateway-openapi-golden`: The gateway's OpenAPI spec matches the checked-in golden file; regenerate it deliberately when the API changes. (blocking: pytest `projects/policyengine-simulation-gateway/tests/test_openapi_golden.py`) +- `sim.contracts.local-integration`: Generated-client integration tests pass against the locally built services. (blocking: CI job `test-integration`) +- `sim.contracts.regenerate-clients`: When public API schemas change, regenerate the clients and run the generated-client integration tests. (doc-only, not checked: follow it yourself) + +### Observability + +- `sim.observability.failures-never-break-requests`: Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. (doc-only, not checked: follow it yourself) +- `sim.observability.header-transport-only`: observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. (doc-only, not checked: follow it yourself) +- `sim.observability.id-is-diagnostic-only`: observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. (doc-only, not checked: follow it yourself) +- `sim.observability.registered-stages-only`: Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. (doc-only, not checked: follow it yourself) + +### Code quality and tests + +- `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself) +- `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format) +- `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself) +- `sim.quality.ruff-lint`: Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet. (advisory: ruff check (E4, E7, E9, F), not blocking) +- `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`) + +### Process and repository hygiene + +- `sim.process.agents-never-bypass`: Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human. (doc-only, not checked: follow it yourself) +- `sim.process.rule-machinery-code-owned`: Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author. (doc-only, not checked: follow it yourself) +- `sim.process.rules-registry`: Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current. (blocking: quality guard `rules-registry`) +- `sim.process.same-repo-draft-prs`: PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. (doc-only, not checked: follow it yourself) +- `sim.process.stacked-prs-full-ci-at-bottom`: In a stack of pull requests, only the lowest unmerged one runs the full PR workflow and the Modal image smoke. Every pr.yml job is needed by the CI complete gate, which fails on the pull requests above it and otherwise passes only when every job passed, so a stack merges bottom-up and nothing merges untested. (blocking: pytest `projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py`) + + diff --git a/changelog_entry.yaml b/changelog_entry.yaml index ff1c790ca..fbc6287fd 100644 --- a/changelog_entry.yaml +++ b/changelog_entry.yaml @@ -1,5 +1,7 @@ - bump: patch changes: + added: + - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md changed: - Added CODEOWNERS for the rule machinery, pinned CI's ruff to 0.15.22 with an explicit rule selection, and documented that agents never bypass branch rules - Ran the Modal image smoke for Stage 12 persistence changes and added policyengine-fastapi to the PR test matrix diff --git a/docs/engineering/skills/README.md b/docs/engineering/skills/README.md index 6b066a9a3..180b4799e 100644 --- a/docs/engineering/skills/README.md +++ b/docs/engineering/skills/README.md @@ -16,3 +16,5 @@ Current skills: - `observability.md`: identifier ownership, HTTP and Modal transport, persistence, registered runtime stages, trace boundaries, and failure isolation. +- `rules-registry.md`: how rules are registered in `rules/registry.toml`, + enforced, proven, and rendered into `docs/generated/rules.md`. diff --git a/docs/engineering/skills/rules-registry.md b/docs/engineering/skills/rules-registry.md new file mode 100644 index 000000000..f94f8cae0 --- /dev/null +++ b/docs/engineering/skills/rules-registry.md @@ -0,0 +1,147 @@ +# Rules Registry + +Read this before adding, changing, relaxing or removing a rule, a guard test, a +checker configuration (ruff, mypy, Pyright, Import Linter), or a CI job that +enforces one. + +Every rule that governs this repository lives in `rules/registry.toml`. The +registry says what each rule requires, how mechanically it is enforced, which +CI job runs it, and whether its checker has been proven to reject a bad example. +`python -m scripts.rules render` turns it into `docs/generated/rules.md` and the +Rules block in `AGENTS.md`; the `rules-registry` quality guard fails CI when the +registry, the checkers and the rendered views disagree. + +## Commands + +```bash +python -m scripts.rules validate # schema and file checks only +python -m scripts.rules render # rewrite rules.md and the AGENTS.md block +python -m scripts.rules guard --base-ref origin/ # everything CI checks +python -m unittest discover -s scripts/rules/tests -t . # the tool's own tests +``` + +All commands use only the standard library, so they run without installing the +project. + +## Adding or changing a rule + +A PR that adds, tightens or relaxes a rule contains all of these: + +1. **The check**: the checker, config or test change. +2. **A registry entry** (fields below). +3. **Proof, appropriate to the tier**: see "Proof". +4. **Zero violations, or a baseline** where the baseline policy allows one. +5. **CI wiring**: a `[[rule.ci]]` reference to the job that runs the check. +6. **Doc update**: explanatory skills docs cite the rule id instead of + restating the rule as unchecked prose. +7. **Regenerated outputs**: run `python -m scripts.rules render` and commit + `docs/generated/rules.md` and `AGENTS.md`. + +If a rule exists only in prose, register it with `status = "doc-only"` so the +gap is visible, rather than leaving it out. + +## Fields + +| Field | Meaning | +| --- | --- | +| `id` | Lowercase dotted id, e.g. `api.v2.services-no-sql`. Never reuse an id. | +| `category` | One of the `[[category]]` ids; controls grouping in the rendered views. | +| `statement` | One plain-English sentence an agent can follow. | +| `rationale` | Path (optionally `#anchor`) to the doc that explains the rule. | +| `scope` | Globs for the files the rule constrains; each must match a file. | +| `tier` | How mechanically the rule can be checked (see "Tiers"). | +| `status` | `enforced`, `ratcheted`, `advisory` or `doc-only`. | +| `autofix` | Whether the enforcer can fix violations itself (e.g. ruff format). | +| `review_by` | Optional date; the guard fails once it passes. | +| `[rule.enforcer]` | What checks the rule; `kind` plus kind-specific fields. | +| `[[rule.ci]]` | `workflow`, `job`, and an `invocation` string that must appear in that job's `run:` text (or its reusable-workflow `uses:`). | +| `[rule.proof]` | Optional; see "Proof". | +| `[rule.baseline]` | Only for `ratcheted` rules: `file` listing tolerated violations, and their `count`. | +| `[[rule.exception]]` | Temporary carve-out: `path`, `reason`, `expires`. | +| `[rule.downgrade]` | Required `reason` when a rule moves from enforced/ratcheted to advisory/doc-only. | + +`blocking` is derived, never declared: a rule blocks merges when its status is +`enforced` or `ratcheted` and its tier is not `judgment`. + +### Enforcer kinds + +| Kind | Fields | Resolved by the guard | +| --- | --- | --- | +| `pytest` | `nodes` (`path::test` or `path::Class::test`), optional `project` (sub-project directory the paths are relative to) | Each test exists, and the CI job's command collects it | +| `quality-guard` | `name`, optional `runner` | The name is in `GUARDS` in `scripts/run_quality_guards.py` | +| `ruff` | `mode` = `format`, `lint` (with `codes`) or `per-file-ignore` (with `pattern`, `codes`) | Codes are selected / the ignore entry exists | +| `mypy` | `paths` | Each path is in `[tool.mypy].files` | +| `workflow-job` | optional `step` | The CI job exists (the job itself is the check) | +| `pyright`, `import-linter` | `config` (+ `contract` for Import Linter) | The config file exists | +| `docs-guard` | `doc`, `marker` | The doc exists | +| `llm-review` | `prompt`, `model` | The prompt exists; judgment tier only | +| `none` | | Nothing; only for `doc-only` and `advisory` rules | + +## Tiers + +| Tier | Covers | Can block? | Proof | +| --- | --- | --- | --- | +| `mechanical` | A pattern is present or not | Yes | A bad fixture is rejected and a good fixture is accepted | +| `metric` | A number crosses a threshold | Yes | Fixtures just below and just above the threshold | +| `judgment` | Needs reading comprehension (LLM review plus human) | No | A labelled eval set with measured precision | + +Judgment rules graduate: when the same judgment finding recurs, turn it into a +mechanical rule. + +## Proof + +A proof shows the checker actually rejects what the rule forbids. Put fixtures +under `rules/fixtures//` and add a test named `test_rule_proof_` +that runs the rule's checker on them. + +| Tier | Proof fields | +| --- | --- | +| `mechanical` | `bad_fixture`, `good_fixture`, `test` (+ optional `project`) | +| `metric` | `below_fixture`, `above_fixture`, `threshold`, `test` | +| `judgment` | `eval_set`, `min_precision`, `measured_precision`, `measured_on` | + +The proof test asserts that the bad fixture is rejected and the good one +accepted. The guard doesn't run it; it checks that the fixtures exist and that a +CI job collects the proof test: + +- For a `pytest`-enforced rule, the proof test must be collected by the rule's + own `[[rule.ci]]` job (for example, under the paths `make test` runs). +- For any other enforcer, name the job that runs the proof test with + `proof.workflow`, `proof.job` and `proof.invocation` (all three or none). + +Where possible, make the bad fixture a real past mistake from git history. +Blocking rules without proof are listed as "unproven" in `rules.md`; they don't +fail the guard yet. + +## Baselines + +- **Ruff is never ratcheted.** Adopting a ruff rule family is one standalone PR + that changes the select list and fixes every resulting violation. Don't use + `--add-noqa` or blanket per-file ignores. +- Other enforcers (mypy, Pyright, Import Linter, custom guards) may start + `ratcheted` with a baseline file of tolerated violations. Baselines may only + shrink: the guard compares counts against the base branch. +- Moving a rule from enforced/ratcheted to advisory/doc-only needs an explicit + `[rule.downgrade] reason`. + +## Hidden rules + +The guard fails when something enforces a rule that the registry doesn't +describe: + +- a `GUARDS` entry in `scripts/run_quality_guards.py`; +- a ruff format check in CI, an explicitly selected ruff code, or a ruff + per-file ignore; +- a `[tool.mypy].files` entry; +- a test matching `[discovery].guard_test_globs` or listed in + `extra_guard_tests`; +- a job in a `[discovery].workflows` file. + +Register it, or list it under `discovery.ignore_jobs` / `discovery.ignore_tests` +with a reason. + +## Keeping the tool in sync + +`scripts/rules/` is byte-identical in policyengine-api and policyengine-sim-api; +see `scripts/rules/SYNC.md`. Change it in both repositories together and bump +`TOOL_VERSION` and each registry's `tool_version`. diff --git a/docs/generated/rules.md b/docs/generated/rules.md new file mode 100644 index 000000000..bb5851ca9 --- /dev/null +++ b/docs/generated/rules.md @@ -0,0 +1,293 @@ +# Rules + +Generated by `python -m scripts.rules render` (scripts/rules v0.1.0, tool sha256 `803fdc2cb511`) from `rules/registry.toml`. Do not edit by hand. + +Every rule that governs `policyengine-sim-api` is listed here with how it is enforced. Blocking rules fail CI. The Gaps section lists what is written down but not yet enforced or proven. To add or change a rule, see `docs/engineering/skills/rules-registry.md`. + +## Summary + +| Tier | enforced | ratcheted | advisory | doc-only | Total | +| --- | ---: | ---: | ---: | ---: | ---: | +| mechanical | 25 | 0 | 1 | 13 | 39 | +| metric | 0 | 0 | 0 | 0 | 0 | +| judgment | 0 | 0 | 0 | 0 | 0 | + +**Blocking:** 25 of 39 rules. + +## Gaps + +### Not enforced (doc-only) + +- `sim.boundaries.gateway-executor-decoupled`: The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. +- `sim.boundaries.unit-tests-need-no-services`: Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. +- `sim.contracts.regenerate-clients`: When public API schemas change, regenerate the clients and run the generated-client integration tests. +- `sim.deploy.image-smoke-covers-every-image-lib`: The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. +- `sim.observability.failures-never-break-requests`: Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. +- `sim.observability.header-transport-only`: observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. +- `sim.observability.id-is-diagnostic-only`: observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. +- `sim.observability.registered-stages-only`: Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. +- `sim.process.agents-never-bypass`: Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human. +- `sim.process.rule-machinery-code-owned`: Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author. +- `sim.process.same-repo-draft-prs`: PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. +- `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. +- `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. + +### Advisory (checked but not blocking) + +- `sim.quality.ruff-lint`: Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet. + +### Unproven (no proof that the checker rejects a bad example) + +- `sim.contracts.clients-publish-deployed-commit` +- `sim.contracts.full-integration-beta-only` +- `sim.contracts.gateway-openapi-golden` +- `sim.deploy.container-context-minimal` +- `sim.deploy.gcloud-without-terraform` +- `sim.deploy.promotion-order-explicit` +- `sim.deploy.revision-tags-valid` +- `sim.deploy.shell-scripts-valid` +- `sim.deploy.smoke-tests-strict` +- `sim.deploy.traffic-changes-exact` +- `sim.deploy.workflow-shell-safety` +- `sim.process.rules-registry` +- `sim.stage12.deployment-scope` +- `sim.stage12.deployment-uses-api-schema` +- `sim.stage12.no-migration-chain` +- `sim.stage12.no-production-user-associations` +- `sim.stage12.persistence-typed-dml-only` +- `sim.stage12.production-lock-has-db-runtime` + +### Ratcheted (existing violations still tolerated) + +- None. + +### Exceptions and review dates + +- None. + +## Service and package boundaries + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.boundaries.gateway-executor-decoupled` | The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract. | mechanical | doc-only | no | nothing | — | — | +| `sim.boundaries.unit-tests-need-no-services` | Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked. | mechanical | doc-only | no | nothing | — | — | + +```mermaid +flowchart LR + r_sim_boundaries_gateway_executor_decoupled("sim.boundaries.gateway-executor-decoupled (doc-only)") + gap_unenforced>"not enforced"] + r_sim_boundaries_unit_tests_need_no_services("sim.boundaries.unit-tests-need-no-services (doc-only)") + r_sim_boundaries_gateway_executor_decoupled -.-> gap_unenforced + r_sim_boundaries_unit_tests_need_no_services -.-> gap_unenforced +``` + +## Stage 12 persistence + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.deployment-uses-api-schema` | Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.no-schema-ddl` | Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | yes | +| `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | +| `sim.stage12.production-lock-has-db-runtime` | The production lock file includes the direct Stage 12 database runtime dependencies. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | + +```mermaid +flowchart LR + r_sim_stage12_deployment_scope("sim.stage12.deployment-scope (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_stage12_deployment_uses_api_schema("sim.stage12.deployment-uses-api-schema (enforced)") + r_sim_stage12_no_migration_chain("sim.stage12.no-migration-chain (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] + r_sim_stage12_no_production_user_associations("sim.stage12.no-production-user-associations (enforced)") + r_sim_stage12_no_schema_ddl("sim.stage12.no-schema-ddl (enforced)") + r_sim_stage12_persistence_typed_dml_only("sim.stage12.persistence-typed-dml-only (enforced)") + e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py["pytest libs/policyengine-stage12-persistence/tests/test_source_boundaries.py"] + r_sim_stage12_production_lock_has_db_runtime("sim.stage12.production-lock-has-db-runtime (enforced)") + r_sim_stage12_deployment_scope --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + r_sim_stage12_deployment_uses_api_schema --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_stage12_no_migration_chain --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test + r_sim_stage12_no_production_user_associations --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + r_sim_stage12_no_schema_ddl --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + r_sim_stage12_persistence_typed_dml_only --> e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py + e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py --> j_CI_pr_yml_test + r_sim_stage12_production_lock_has_db_runtime --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py +``` + +## Deployment pipeline + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.docker-images-build` | The simulation entry and executor Docker images build. | mechanical | enforced | yes | CI job `docker-build` | `pr.yml:docker-build` | — | +| `sim.deploy.gcloud-without-terraform` | Deployment uses the gcloud-based workflow, not Terraform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.image-smoke-covers-every-image-lib` | The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. | mechanical | doc-only | no | nothing | — | — | +| `sim.deploy.modal-image-smoke` | App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes. | mechanical | enforced | yes | CI job `image-smoke` | `pr-image-smoke.yml:image-smoke` | — | +| `sim.deploy.promotion-order-explicit` | Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.revision-tags-valid` | Cloud Run revision tags are valid on a workflow's first run and fit the existing service names. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.shell-scripts-valid` | Deployment shell scripts have valid shell syntax. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.smoke-tests-strict` | Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.traffic-changes-exact` | Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | + +```mermaid +flowchart LR + r_sim_deploy_container_context_minimal("sim.deploy.container-context-minimal (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_deploy_docker_images_build("sim.deploy.docker-images-build (enforced)") + e_CI_job_docker_build["CI job docker-build"] + j_CI_pr_yml_docker_build["CI pr.yml:docker-build"] + r_sim_deploy_gcloud_without_terraform("sim.deploy.gcloud-without-terraform (enforced)") + r_sim_deploy_image_smoke_covers_every_image_lib("sim.deploy.image-smoke-covers-every-image-lib (doc-only)") + gap_unenforced>"not enforced"] + r_sim_deploy_modal_image_smoke("sim.deploy.modal-image-smoke (enforced)") + e_CI_job_image_smoke["CI job image-smoke"] + j_CI_pr_image_smoke_yml_image_smoke["CI pr-image-smoke.yml:image-smoke"] + r_sim_deploy_promotion_order_explicit("sim.deploy.promotion-order-explicit (enforced)") + r_sim_deploy_revision_tags_valid("sim.deploy.revision-tags-valid (enforced)") + r_sim_deploy_shell_scripts_valid("sim.deploy.shell-scripts-valid (enforced)") + r_sim_deploy_smoke_tests_strict("sim.deploy.smoke-tests-strict (enforced)") + r_sim_deploy_traffic_changes_exact("sim.deploy.traffic-changes-exact (enforced)") + r_sim_deploy_workflow_shell_safety("sim.deploy.workflow-shell-safety (enforced)") + r_sim_deploy_container_context_minimal --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + r_sim_deploy_docker_images_build --> e_CI_job_docker_build + e_CI_job_docker_build --> j_CI_pr_yml_docker_build + r_sim_deploy_gcloud_without_terraform --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_image_smoke_covers_every_image_lib -.-> gap_unenforced + r_sim_deploy_modal_image_smoke --> e_CI_job_image_smoke + e_CI_job_image_smoke --> j_CI_pr_image_smoke_yml_image_smoke + r_sim_deploy_promotion_order_explicit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_revision_tags_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_shell_scripts_valid --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_smoke_tests_strict --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_traffic_changes_exact --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_deploy_workflow_shell_safety --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py +``` + +## API contracts and integration + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.contracts.clients-publish-deployed-commit` | Client publication checks out the commit that was actually deployed. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.full-integration-beta-only` | The complete integration suite runs against beta deployments only, never production. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.gateway-openapi-golden` | The gateway's OpenAPI spec matches the checked-in golden file; regenerate it deliberately when the API changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-gateway/tests/test_openapi_golden.py` | `pr.yml:test` | **unproven** | +| `sim.contracts.local-integration` | Generated-client integration tests pass against the locally built services. | mechanical | enforced | yes | CI job `test-integration` | `pr.yml:test-integration` | — | +| `sim.contracts.regenerate-clients` | When public API schemas change, regenerate the clients and run the generated-client integration tests. | mechanical | doc-only | no | nothing | — | — | + +```mermaid +flowchart LR + r_sim_contracts_clients_publish_deployed_commit("sim.contracts.clients-publish-deployed-commit (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_contracts_full_integration_beta_only("sim.contracts.full-integration-beta-only (enforced)") + r_sim_contracts_gateway_openapi_golden("sim.contracts.gateway-openapi-golden (enforced)") + e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py["pytest projects/policyengine-simulation-gateway/tests/test_openapi_golden.py"] + r_sim_contracts_local_integration("sim.contracts.local-integration (enforced)") + e_CI_job_test_integration["CI job test-integration"] + j_CI_pr_yml_test_integration["CI pr.yml:test-integration"] + r_sim_contracts_regenerate_clients("sim.contracts.regenerate-clients (doc-only)") + gap_unenforced>"not enforced"] + r_sim_contracts_clients_publish_deployed_commit --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test + r_sim_contracts_full_integration_beta_only --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py + r_sim_contracts_gateway_openapi_golden --> e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py + e_pytest_projects_policyengine_simulation_gateway_tests_test_openapi_golden_py --> j_CI_pr_yml_test + r_sim_contracts_local_integration --> e_CI_job_test_integration + e_CI_job_test_integration --> j_CI_pr_yml_test_integration + r_sim_contracts_regenerate_clients -.-> gap_unenforced +``` + +## Observability + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.observability.failures-never-break-requests` | Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation. | mechanical | doc-only | no | nothing | — | — | +| `sim.observability.header-transport-only` | observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one. | mechanical | doc-only | no | nothing | — | — | +| `sim.observability.id-is-diagnostic-only` | observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic. | mechanical | doc-only | no | nothing | — | — | +| `sim.observability.registered-stages-only` | Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals. | mechanical | doc-only | no | nothing | — | — | + +```mermaid +flowchart LR + r_sim_observability_failures_never_break_requests("sim.observability.failures-never-break-requests (doc-only)") + gap_unenforced>"not enforced"] + r_sim_observability_header_transport_only("sim.observability.header-transport-only (doc-only)") + r_sim_observability_id_is_diagnostic_only("sim.observability.id-is-diagnostic-only (doc-only)") + r_sim_observability_registered_stages_only("sim.observability.registered-stages-only (doc-only)") + r_sim_observability_failures_never_break_requests -.-> gap_unenforced + r_sim_observability_header_transport_only -.-> gap_unenforced + r_sim_observability_id_is_diagnostic_only -.-> gap_unenforced + r_sim_observability_registered_stages_only -.-> gap_unenforced +``` + +## Code quality and tests + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — | +| `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | +| `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | +| `sim.quality.ruff-lint` | Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet. | mechanical | advisory | no | ruff check (E4, E7, E9, F) | — | — | +| `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | + +```mermaid +flowchart LR + r_sim_quality_every_project_tested("sim.quality.every-project-tested (doc-only)") + gap_unenforced>"not enforced"] + r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)") + e_ruff_format["ruff format"] + j_CI_pr_yml_lint["CI pr.yml:lint"] + r_sim_quality_ruff_format_tests("sim.quality.ruff-format-tests (doc-only)") + r_sim_quality_ruff_lint("sim.quality.ruff-lint (advisory)") + e_ruff_check__E4__E7__E9__F_["ruff check (E4, E7, E9, F)"] + local_only>"not run in CI"] + r_sim_quality_tests_pass("sim.quality.tests-pass (enforced)") + e_CI_job_test["CI job test"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_quality_every_project_tested -.-> gap_unenforced + r_sim_quality_ruff_format_src --> e_ruff_format + e_ruff_format --> j_CI_pr_yml_lint + r_sim_quality_ruff_format_tests -.-> gap_unenforced + r_sim_quality_ruff_lint --> e_ruff_check__E4__E7__E9__F_ + e_ruff_check__E4__E7__E9__F_ -.-> local_only + r_sim_quality_tests_pass --> e_CI_job_test + e_CI_job_test --> j_CI_pr_yml_test +``` + +## Process and repository hygiene + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.process.agents-never-bypass` | Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human. | mechanical | doc-only | no | nothing | — | — | +| `sim.process.rule-machinery-code-owned` | Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author. | mechanical | doc-only | no | nothing | — | — | +| `sim.process.rules-registry` | Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current. | mechanical | enforced | yes | quality guard `rules-registry` | `pr.yml:quality-guards` | **unproven** | +| `sim.process.same-repo-draft-prs` | PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'. | mechanical | doc-only | no | nothing | — | — | +| `sim.process.stacked-prs-full-ci-at-bottom` | In a stack of pull requests, only the lowest unmerged one runs the full PR workflow and the Modal image smoke. Every pr.yml job is needed by the CI complete gate, which fails on the pull requests above it and otherwise passes only when every job passed, so a stack merges bottom-up and nothing merges untested. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py` | `pr.yml:test`, `pr.yml:ci-complete` | yes | + +```mermaid +flowchart LR + r_sim_process_agents_never_bypass("sim.process.agents-never-bypass (doc-only)") + gap_unenforced>"not enforced"] + r_sim_process_rule_machinery_code_owned("sim.process.rule-machinery-code-owned (doc-only)") + r_sim_process_rules_registry("sim.process.rules-registry (enforced)") + e_quality_guard_rules_registry["quality guard rules-registry"] + j_CI_pr_yml_quality_guards["CI pr.yml:quality-guards"] + r_sim_process_same_repo_draft_prs("sim.process.same-repo-draft-prs (doc-only)") + r_sim_process_stacked_prs_full_ci_at_bottom("sim.process.stacked-prs-full-ci-at-bottom (enforced)") + e_pytest_projects_policyengine_simulation_executor_tests_test_ci_complete_gate_py["pytest projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + j_CI_pr_yml_ci_complete["CI pr.yml:ci-complete"] + r_sim_process_agents_never_bypass -.-> gap_unenforced + r_sim_process_rule_machinery_code_owned -.-> gap_unenforced + r_sim_process_rules_registry --> e_quality_guard_rules_registry + e_quality_guard_rules_registry --> j_CI_pr_yml_quality_guards + r_sim_process_same_repo_draft_prs -.-> gap_unenforced + r_sim_process_stacked_prs_full_ci_at_bottom --> e_pytest_projects_policyengine_simulation_executor_tests_test_ci_complete_gate_py + e_pytest_projects_policyengine_simulation_executor_tests_test_ci_complete_gate_py --> j_CI_pr_yml_test + e_pytest_projects_policyengine_simulation_executor_tests_test_ci_complete_gate_py --> j_CI_pr_yml_ci_complete +``` diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py index 8002614f7..abc4a3ea1 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py @@ -31,6 +31,15 @@ def test_simulation_runtimes_contain_no_schema_ddl() -> None: assert violations == [] +def test_rule_proof_stage12_no_schema_ddl() -> None: + """Proof for rule sim.stage12.no-schema-ddl (rules/registry.toml).""" + fixtures = REPOSITORY_ROOT / "rules/fixtures/sim.stage12.no-schema-ddl" + bad = (fixtures / "bad_stage12_runtime.py").read_text(encoding="utf-8") + good = (fixtures / "good_stage12_runtime.py").read_text(encoding="utf-8") + assert DDL_PATTERN.search(bad) is not None + assert DDL_PATTERN.search(good) is None + + def test_simulation_repository_has_no_schema_migration_chain() -> None: assert not list(REPOSITORY_ROOT.glob("alembic*.ini")) assert not list(REPOSITORY_ROOT.rglob("alembic/versions")) diff --git a/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py b/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py index b1f45dfd2..798704db2 100644 --- a/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py +++ b/projects/policyengine-simulation-executor/tests/test_ci_complete_gate.py @@ -63,6 +63,19 @@ def test_pr_workflow_is_gated() -> None: assert gate_violations(workflow) == [] +def test_image_smoke_runs_only_at_the_bottom_of_a_stack() -> None: + smoke = yaml.safe_load( + (REPO / ".github" / "workflows" / "pr-image-smoke.yml").read_text() + ) + stack_condition = ( + "(!github.event.pull_request.stack || " + "github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref)" + ) + + for job_id, job in smoke["jobs"].items(): + assert stack_condition in job.get("if", ""), job_id + + def test_gate_check_rejects_an_ungated_job() -> None: workflow = { "jobs": { @@ -78,6 +91,18 @@ def test_gate_check_rejects_an_ungated_job() -> None: ] +def test_rule_proof_gate_check() -> None: + fixtures = REPO / "rules" / "fixtures" / "sim.process.stacked-prs-full-ci-at-bottom" + bad = yaml.safe_load((fixtures / "bad_pr.yml").read_text()) + good = yaml.safe_load((fixtures / "good_pr.yml").read_text()) + + assert gate_violations(bad) == [ + f"{GATE_JOB} does not need test", + f"test must have if: {FULL_CI_CONDITION}", + ] + assert gate_violations(good) == [] + + def _run_gate( job_results: dict[str, str], *, diff --git a/rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/bad_pr.yml b/rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/bad_pr.yml new file mode 100644 index 000000000..56ec40675 --- /dev/null +++ b/rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/bad_pr.yml @@ -0,0 +1,26 @@ +# Proof fixture: a PR workflow where a job escapes the CI complete gate. +# "test" has no stack condition and the gate doesn't need it, so on a stacked +# pull request it would still run, and its failure would never block a merge. +name: Pull request + +on: pull_request + +jobs: + ci-complete: + name: CI complete + if: ${{ always() }} + needs: [lint] + runs-on: ubuntu-latest + steps: + - run: bash .github/scripts/check_ci_complete.sh + lint: + name: Lint + if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }} + runs-on: ubuntu-latest + steps: + - run: ruff format --check . + test: + name: Test + runs-on: ubuntu-latest + steps: + - run: uv run pytest tests/ diff --git a/rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/good_pr.yml b/rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/good_pr.yml new file mode 100644 index 000000000..a1ddd51c8 --- /dev/null +++ b/rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/good_pr.yml @@ -0,0 +1,27 @@ +# Proof fixture: a PR workflow where every job is gated. Root jobs carry the +# stack condition, jobs that need them are skipped with them, and the CI +# complete gate needs every job. +name: Pull request + +on: pull_request + +jobs: + ci-complete: + name: CI complete + if: ${{ always() }} + needs: [lint, test] + runs-on: ubuntu-latest + steps: + - run: bash .github/scripts/check_ci_complete.sh + lint: + name: Lint + if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }} + runs-on: ubuntu-latest + steps: + - run: ruff format --check . + test: + name: Test + needs: lint + runs-on: ubuntu-latest + steps: + - run: uv run pytest tests/ diff --git a/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py b/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py new file mode 100644 index 000000000..f6b4b7262 --- /dev/null +++ b/rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py @@ -0,0 +1,7 @@ +"""Proof fixture: a Stage 12 runtime that creates its own table. The rule must reject it.""" + +from sqlalchemy import text + + +def ensure_comparison_table(connection) -> None: + connection.execute(text("CREATE TABLE stage12_evaluation_reports (id uuid)")) diff --git a/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py b/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py new file mode 100644 index 000000000..6dddf8588 --- /dev/null +++ b/rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py @@ -0,0 +1,7 @@ +"""Proof fixture: a Stage 12 runtime that writes rows with typed DML. The rule must accept it.""" + +from sqlalchemy import insert + + +def record_report(connection, reports_table, report_id: str) -> None: + connection.execute(insert(reports_table).values(id=report_id)) diff --git a/rules/registry.toml b/rules/registry.toml new file mode 100644 index 000000000..c6c339387 --- /dev/null +++ b/rules/registry.toml @@ -0,0 +1,717 @@ +# Rules registry for policyengine-sim-api. +# +# Every rule that governs this repository is declared here: what it says, how +# mechanically it is enforced, where it runs in CI, and whether a checker has +# been proven to reject a bad example. `python -m scripts.rules guard` keeps this +# file honest; `python -m scripts.rules render` regenerates +# docs/generated/rules.md and the Rules block in AGENTS.md. +# +# How to add or change a rule: docs/engineering/skills/rules-registry.md + +schema_version = 1 +repo = "policyengine-sim-api" +tool_version = "0.1.0" + +[[category]] +id = "boundaries" +title = "Service and package boundaries" + +[[category]] +id = "stage12" +title = "Stage 12 persistence" + +[[category]] +id = "deployment" +title = "Deployment pipeline" + +[[category]] +id = "contracts" +title = "API contracts and integration" + +[[category]] +id = "observability" +title = "Observability" + +[[category]] +id = "quality" +title = "Code quality and tests" + +[[category]] +id = "process" +title = "Process and repository hygiene" + +[discovery] +workflows = [".github/workflows/pr.yml", ".github/workflows/pr-image-smoke.yml"] +guard_test_globs = [ + "projects/*/tests/test_*boundar*.py", + "libs/*/tests/test_*boundar*.py", + "projects/*/tests/test_*schema_authority*.py", + "projects/policyengine-simulation-entry/tests/test_deployment_assets.py", + "projects/policyengine-simulation-gateway/tests/test_openapi_golden.py", +] + +# --- Service and package boundaries ---------------------------------------- + +[[rule]] +id = "sim.boundaries.gateway-executor-decoupled" +category = "boundaries" +statement = "The gateway and executor never import each other; they communicate only through the models and state helpers in policyengine-simulation-contract." +rationale = "libs/policyengine-simulation-contract/README.md" +scope = ["projects/policyengine-simulation-gateway/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.boundaries.unit-tests-need-no-services" +category = "boundaries" +statement = "Unit tests never require real network credentials, Modal, Hugging Face, GCP or deployed services; those seams are mocked." +rationale = "docs/engineering/skills/testing.md#dependency-boundaries" +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- Stage 12 persistence -------------------------------------------------- + +[[rule]] +id = "sim.stage12.no-schema-ddl" +category = "stage12" +statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" +good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +project = "projects/policyengine-simulation-entry" +test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" + +[[rule]] +id = "sim.stage12.no-migration-chain" +category = "stage12" +statement = "This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_stage12_schema_authority.py::test_simulation_repository_has_no_schema_migration_chain"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.no-production-user-associations" +category = "stage12" +statement = "Stage 12 runtime modules never create production user associations (user_simulations or user_reports)." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/*/src/**/stage12_*.py", "libs/policyengine-stage12-persistence/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_stage12_schema_authority.py::test_stage12_runtime_cannot_create_production_user_associations"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.persistence-typed-dml-only" +category = "stage12" +statement = "The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["libs/policyengine-stage12-persistence/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-stage12-persistence" +nodes = ["tests/test_source_boundaries.py::test_persistence_package_contains_no_raw_sql_or_schema_operations"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.deployment-uses-api-schema" +category = "stage12" +statement = "Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/policyengine-simulation-entry/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_stage12_deployment_uses_the_api_owned_schema_directly", + "tests/test_deployment_assets.py::test_stage12_runtime_uses_typed_dml_without_raw_sql_or_schema_changes", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.deployment-scope" +category = "stage12" +statement = "Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources; the Stage 12 worker can request a Google OIDC token." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_main_deployment_automatically_deploys_stage12_in_both_environments", + "tests/test_deployment_assets.py::test_stage12_only_deployment_runs_candidate_tests_and_requested_promotion", + "tests/test_deployment_assets.py::test_stage12_only_deployment_cannot_redeploy_existing_modal_resources", + "tests/test_deployment_assets.py::test_stage12_worker_deployment_can_request_a_google_oidc_token", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.stage12.production-lock-has-db-runtime" +category = "stage12" +statement = "The production lock file includes the direct Stage 12 database runtime dependencies." +rationale = "libs/policyengine-stage12-persistence/README.md" +scope = ["projects/policyengine-simulation-entry/uv.lock"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_production_lock_contains_the_direct_stage12_database_runtime"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +# --- Deployment pipeline --------------------------------------------------- + +[[rule]] +id = "sim.deploy.shell-scripts-valid" +category = "deployment" +statement = "Deployment shell scripts have valid shell syntax." +rationale = "docs/operations" +scope = [".github/scripts/*.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_deployment_scripts_have_valid_shell_syntax"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.gcloud-without-terraform" +category = "deployment" +statement = "Deployment uses the gcloud-based workflow, not Terraform." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_deployment_uses_gcloud_workflow_without_terraform"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.workflow-shell-safety" +category = "deployment" +statement = "Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_cloud_run_deployment_does_not_interpolate_github_values_in_shell", + "tests/test_deployment_assets.py::test_cloud_run_deployment_escapes_environment_and_pins_secret_versions", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.revision-tags-valid" +category = "deployment" +statement = "Cloud Run revision tags are valid on a workflow's first run and fit the existing service names." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_cloud_run_revision_tags_are_valid_on_the_first_workflow_run", + "tests/test_deployment_assets.py::test_revision_tags_fit_existing_cloud_run_service_names", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.promotion-order-explicit" +category = "deployment" +statement = "Full-stack promotion order is explicit, and the reusable deployment rejects unexpectedly skipped phases." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_full_stack_promotion_order_is_explicit", + "tests/test_deployment_assets.py::test_reusable_deployment_rejects_unexpected_skipped_phases", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.smoke-tests-strict" +category = "deployment" +statement = "Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge." +rationale = "docs/operations" +scope = [".github/scripts/*.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_candidate_smoke_rejects_a_revision_mismatch_without_retrying", + "tests/test_deployment_assets.py::test_stable_smoke_rejects_an_unexpected_revision_without_retrying", + "tests/test_deployment_assets.py::test_stable_smoke_retries_the_previous_revision_until_target_is_served", + "tests/test_deployment_assets.py::test_stable_smoke_fails_when_the_previous_revision_does_not_converge", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.traffic-changes-exact" +category = "deployment" +statement = "Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion." +rationale = "docs/operations" +scope = [".github/scripts/*.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_traffic_changes_are_exact_revision_validated_and_reversible", + "tests/test_deployment_assets.py::test_traffic_change_refuses_an_intervening_promotion", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.container-context-minimal" +category = "deployment" +statement = "Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform." +rationale = "docs/operations" +scope = ["projects/policyengine-simulation-entry/Dockerfile"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = [ + "tests/test_deployment_assets.py::test_container_context_excludes_local_environments_and_unrelated_projects", + "tests/test_deployment_assets.py::test_entrypoint_container_uses_requested_target_platform", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.deploy.docker-images-build" +category = "deployment" +statement = "The simulation entry and executor Docker images build." +rationale = "docs/operations" +scope = ["projects/policyengine-simulation-entry/Dockerfile", "projects/policyengine-simulation-executor/Dockerfile"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "docker-build" +invocation = "docker/build-push-action" + +[[rule]] +id = "sim.deploy.modal-image-smoke" +category = "deployment" +statement = "App entrypoints import inside the real Modal images before merge, whenever a library baked into an image changes." +rationale = ".github/workflows/pr-image-smoke.yml" +scope = ["projects/policyengine-simulation-executor/src/modal/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr-image-smoke.yml" +job = "image-smoke" +invocation = "modal-image-smoke.sh" + +[[rule]] +id = "sim.deploy.image-smoke-covers-every-image-lib" +category = "deployment" +statement = "The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke." +rationale = ".github/workflows/pr-image-smoke.yml" +scope = [".github/workflows/pr-image-smoke.yml"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- API contracts and integration ---------------------------------------- + +[[rule]] +id = "sim.contracts.gateway-openapi-golden" +category = "contracts" +statement = "The gateway's OpenAPI spec matches the checked-in golden file; regenerate it deliberately when the API changes." +rationale = "projects/policyengine-simulation-gateway/tests/test_openapi_golden.py" +scope = ["projects/policyengine-simulation-gateway/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-gateway" +nodes = ["tests/test_openapi_golden.py::test_openapi_spec_matches_golden"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.contracts.regenerate-clients" +category = "contracts" +statement = "When public API schemas change, regenerate the clients and run the generated-client integration tests." +rationale = "docs/engineering/skills/testing.md#dependency-boundaries" +scope = ["projects/policyengine-simulation-entry/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.contracts.local-integration" +category = "contracts" +statement = "Generated-client integration tests pass against the locally built services." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/policyengine-apis-integ/tests/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test-integration" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.contracts.full-integration-beta-only" +category = "contracts" +statement = "The complete integration suite runs against beta deployments only, never production." +rationale = "docs/engineering/skills/testing.md" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_complete_integration_suite_is_configured_for_beta_only"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.contracts.clients-publish-deployed-commit" +category = "contracts" +statement = "Client publication checks out the commit that was actually deployed." +rationale = "docs/operations" +scope = [".github/workflows/*.yml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-entry" +nodes = ["tests/test_deployment_assets.py::test_client_publication_checks_out_the_deployed_commit"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +# --- Observability --------------------------------------------------------- + +[[rule]] +id = "sim.observability.id-is-diagnostic-only" +category = "observability" +statement = "observability_id has diagnostic meaning only: never use it for idempotency, authorization, database identity, routing or business logic." +rationale = "docs/engineering/skills/observability.md#identifier-ownership" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.observability.header-transport-only" +category = "observability" +statement = "observability_id travels only in the X-PolicyEngine-Observability-Id header, never in a simulation JSON body, and polling never invents one." +rationale = "docs/engineering/skills/observability.md#http-transport" +scope = ["projects/*/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.observability.registered-stages-only" +category = "observability" +statement = "Span names come from the stage plans in policyengine_simulation_observability.stages; runtime code never adds span-name string literals." +rationale = "docs/engineering/skills/observability.md#runtime-stages" +scope = ["projects/*/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.observability.failures-never-break-requests" +category = "observability" +statement = "Invalid observability configuration fails at build or deploy time, and telemetry failures at runtime never fail a request or computation." +rationale = "docs/engineering/skills/observability.md#failure-behavior" +scope = ["libs/policyengine-simulation-observability/src/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- Code quality and tests ------------------------------------------------ + +[[rule]] +id = "sim.quality.ruff-format-src" +category = "quality" +statement = "Source code under projects/*/src and libs/*/src is formatted with ruff format." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = true +[rule.enforcer] +kind = "ruff" +mode = "format" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "lint" +invocation = "ruff format --check" + +[[rule]] +id = "sim.quality.ruff-format-tests" +category = "quality" +statement = "Test code is formatted with ruff format too; CI currently checks only src directories." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = true +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.quality.ruff-lint" +category = "quality" +statement = "Python code passes ruff check with the rules selected in ruff.toml (ruff's defaults, made explicit); CI does not run ruff check yet." +rationale = "docs/engineering/skills/testing.md#common-commands" +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +tier = "mechanical" +status = "advisory" +autofix = true +[rule.enforcer] +kind = "ruff" +mode = "lint" +codes = ["E4", "E7", "E9", "F"] + +[[rule]] +id = "sim.quality.tests-pass" +category = "quality" +statement = "Each project's unit tests pass in its own locked environment." +rationale = "docs/engineering/skills/testing.md#canonical-layout" +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "workflow-job" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" + +[[rule]] +id = "sim.quality.every-project-tested" +category = "quality" +statement = "Every project and library with unit tests runs them in the PR test matrix." +rationale = "docs/engineering/skills/testing.md#canonical-layout" +scope = ["projects/*/tests/**/*.py", "libs/*/tests/**/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +# --- Process and repository hygiene ---------------------------------------- + +[[rule]] +id = "sim.process.rules-registry" +category = "process" +statement = "Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current." +rationale = "docs/engineering/skills/rules-registry.md" +scope = ["rules/registry.toml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "rules-registry" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" + +[[rule]] +id = "sim.process.stacked-prs-full-ci-at-bottom" +category = "process" +statement = "In a stack of pull requests, only the lowest unmerged one runs the full PR workflow and the Modal image smoke. Every pr.yml job is needed by the CI complete gate, which fails on the pull requests above it and otherwise passes only when every job passed, so a stack merges bottom-up and nothing merges untested." +rationale = "docs/engineering/skills/github-prs.md#stacked-prs-and-ci" +scope = [".github/workflows/pr.yml", ".github/workflows/pr-image-smoke.yml", ".github/scripts/check_ci_complete.sh"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/policyengine-simulation-executor" +nodes = ["tests/test_ci_complete_gate.py::test_pr_workflow_is_gated", "tests/test_ci_complete_gate.py::test_image_smoke_runs_only_at_the_bottom_of_a_stack"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run pytest tests/" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "ci-complete" +invocation = "check_ci_complete.sh" +[rule.proof] +bad_fixture = "rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/bad_pr.yml" +good_fixture = "rules/fixtures/sim.process.stacked-prs-full-ci-at-bottom/good_pr.yml" +project = "projects/policyengine-simulation-executor" +test = "tests/test_ci_complete_gate.py::test_rule_proof_gate_check" + +[[rule]] +id = "sim.process.rule-machinery-code-owned" +category = "process" +statement = "Changes to the rule machinery (rules/, scripts/rules/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author." +rationale = ".github/CODEOWNERS" +scope = [".github/CODEOWNERS"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.process.agents-never-bypass" +category = "process" +statement = "Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human." +rationale = "docs/engineering/skills/github-prs.md#branch-rules" +scope = ["AGENTS.md"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" + +[[rule]] +id = "sim.process.same-repo-draft-prs" +category = "process" +statement = "PRs are opened as drafts from branches in PolicyEngine/policyengine-sim-api, never from forks, and start with 'Fixes #'." +rationale = "docs/engineering/skills/github-prs.md" +scope = [".github/pull_request_template.md"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" diff --git a/scripts/rules/SYNC.md b/scripts/rules/SYNC.md new file mode 100644 index 000000000..4a3cf44ef --- /dev/null +++ b/scripts/rules/SYNC.md @@ -0,0 +1,19 @@ +# Keeping `scripts/rules/` in sync + +This directory is byte-identical in two repositories: + +- `PolicyEngine/policyengine-api` (canonical copy) +- `PolicyEngine/policyengine-sim-api` + +When you change it: + +1. Make the change in policyengine-api first. +2. Copy the whole directory to policyengine-sim-api unchanged. +3. Bump `TOOL_VERSION` in `__init__.py`, and `tool_version` in each repository's + `rules/registry.toml`. +4. Run `python -m scripts.rules render` in both repositories. The header of + `docs/generated/rules.md` shows a hash of the tool's source, so reviewers can + confirm both copies match. + +Repository-specific content belongs in `rules/registry.toml`, never in this +directory. diff --git a/scripts/rules/__init__.py b/scripts/rules/__init__.py new file mode 100644 index 000000000..09bd68688 --- /dev/null +++ b/scripts/rules/__init__.py @@ -0,0 +1,12 @@ +"""Rules registry tooling. + +The registry in ``rules/registry.toml`` is the single list of rules that govern +this repository. This package validates it, checks that every rule is really +enforced where it claims to be, and renders the human- and agent-facing views. + +This directory is kept byte-identical across policyengine-api and +policyengine-sim-api; see ``SYNC.md``. It must stay standard-library only so it +can run in CI jobs that install no dependencies. +""" + +TOOL_VERSION = "0.1.0" diff --git a/scripts/rules/__main__.py b/scripts/rules/__main__.py new file mode 100644 index 000000000..96e63b9b7 --- /dev/null +++ b/scripts/rules/__main__.py @@ -0,0 +1,5 @@ +"""Entry point for ``python -m scripts.rules``.""" + +from scripts.rules.cli import main + +raise SystemExit(main()) diff --git a/scripts/rules/baseline.py b/scripts/rules/baseline.py new file mode 100644 index 000000000..9728f904b --- /dev/null +++ b/scripts/rules/baseline.py @@ -0,0 +1,82 @@ +"""Compare ratchets against the base branch: baselines may only shrink.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from scripts.rules.load import REGISTRY_PATH, load_text +from scripts.rules.model import Registry, Status, Violation + + +class BaseUnavailable(Exception): + """The base ref could not be read.""" + + +def default_base_ref() -> str | None: + base = os.environ.get("GITHUB_BASE_REF") + return f"origin/{base}" if base else None + + +def load_base_registry(root: Path, ref: str) -> Registry | None: + """The registry as it is on ``ref``, or None if the base has no registry yet.""" + verify = subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + cwd=root, + capture_output=True, + text=True, + ) + if verify.returncode != 0: + raise BaseUnavailable(ref) + shown = subprocess.run( + ["git", "show", f"{ref}:{REGISTRY_PATH.as_posix()}"], + cwd=root, + capture_output=True, + text=True, + ) + if shown.returncode != 0: + return None + registry, _ = load_text(shown.stdout) + return registry + + +def check_baselines( + current: Registry, base: Registry +) -> tuple[list[Violation], list[str]]: + """Return (violations, notes).""" + violations: list[Violation] = [] + notes: list[str] = [] + current_ids = {rule.id for rule in current.rules} + for old in base.rules: + new = current.rule(old.id) + if new is None: + notes.append(f"rule {old.id} was removed") + continue + if old.baseline and new.baseline and new.baseline.count > old.baseline.count: + violations.append( + Violation( + new.id, + "baseline", + f"baseline grew from {old.baseline.count} to {new.baseline.count}", + "fix the new violations instead of adding them to the baseline", + ) + ) + weaker = (Status.ADVISORY, Status.DOC_ONLY) + if old.status in (Status.ENFORCED, Status.RATCHETED) and new.status in weaker: + if new.downgrade_reason: + notes.append( + f"rule {new.id} downgraded to {new.status}: {new.downgrade_reason}" + ) + else: + violations.append( + Violation( + new.id, + "baseline", + f"status weakened from {old.status} to {new.status}", + 'add [rule.downgrade] reason = "..." so the downgrade is explicit and reviewable', + ) + ) + for rule_id in sorted(current_ids - {rule.id for rule in base.rules}): + notes.append(f"rule {rule_id} is new") + return violations, notes diff --git a/scripts/rules/cli.py b/scripts/rules/cli.py new file mode 100644 index 000000000..e5fd94fda --- /dev/null +++ b/scripts/rules/cli.py @@ -0,0 +1,54 @@ +"""Command line: ``python -m scripts.rules {validate,render,guard}``.""" + +from __future__ import annotations + +import argparse +from datetime import date +from pathlib import Path + +from scripts.rules.guard import REPO_ROOT, run_guard +from scripts.rules.load import load_registry +from scripts.rules.render import write_outputs +from scripts.rules.resolve import clear_caches +from scripts.rules.validate import validate + + +def _print(violations: list, notes: list[str] | None = None) -> int: + for note in notes or []: + print(f"note: {note}") + for violation in violations: + print(f" - {violation}") + print("rules: failed" if violations else "rules: passed") + return 1 if violations else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m scripts.rules", description=__doc__ + ) + parser.add_argument("command", choices=["validate", "render", "guard"]) + parser.add_argument("--root", type=Path, default=REPO_ROOT, help="repository root") + parser.add_argument( + "--base-ref", + help="git ref to compare baselines against (default: origin/$GITHUB_BASE_REF)", + ) + args = parser.parse_args(argv) + root = args.root.resolve() + + if args.command == "guard": + result = run_guard(root, base_ref=args.base_ref) + return _print(result.violations, result.notes) + + clear_caches() + registry, violations = load_registry(root) + if registry is None: + return _print(violations) + violations += validate(registry, root, date.today()) + if args.command == "render": + if violations: + print("render refused: fix the registry first") + return _print(violations) + for path in write_outputs(registry, root): + print(f"wrote {path.relative_to(root)}") + return 0 + return _print(violations) diff --git a/scripts/rules/discover.py b/scripts/rules/discover.py new file mode 100644 index 000000000..c7c7e42d9 --- /dev/null +++ b/scripts/rules/discover.py @@ -0,0 +1,164 @@ +"""Find rules that are enforced somewhere but missing from the registry. + +A check that isn't registered is a hidden rule: nobody can see it in +``rules.md`` and agents aren't told about it. This scans the places checks live +today and requires each one to be referenced by a rule, or ignored with a +reason in ``[discovery]``. +""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.rules.model import Kind, Registry, Violation +from scripts.rules.resolve import ( + DEFAULT_RUNNER, + guard_names, + mypy_files, + root_relative_nodes, + ruff_lint_config, + split_node, + test_names, + workflow_jobs, +) + + +def _hidden(message: str, fix: str) -> Violation: + return Violation(None, "hidden-rule", message, fix) + + +def _registered_nodes(registry: Registry) -> set[str]: + nodes = set() + for rule in registry.rules: + if rule.enforcer.kind is Kind.PYTEST: + for node in root_relative_nodes(rule): + path, name = split_node(node) + nodes.add(f"{path}::{name}") + if rule.proof is not None and isinstance(rule.proof.fields.get("test"), str): + project = rule.proof.fields.get("project") + prefix = f"{project.rstrip('/')}/" if isinstance(project, str) else "" + path, name = split_node(prefix + rule.proof.fields["test"]) + nodes.add(f"{path}::{name}") + return nodes + + +def _discovered_tests(registry: Registry, root: Path) -> set[str]: + found = set() + for pattern in registry.discovery.guard_test_globs: + for path in sorted(root.glob(pattern)): + names = test_names(path) + relative = path.relative_to(root).as_posix() + for name in names or (): + found.add(f"{relative}::{name}") + for node in registry.discovery.extra_guard_tests: + path, name = split_node(node) + found.add(f"{path}::{name}") + return found + + +def discover_hidden(registry: Registry, root: Path) -> list[Violation]: + violations = [] + rules = registry.rules + + # Quality guards registered in the runner. + names = guard_names(root, DEFAULT_RUNNER) or [] + registered = { + r.enforcer.get("name") for r in rules if r.enforcer.kind is Kind.QUALITY_GUARD + } + for name in names: + if name not in registered: + violations.append( + _hidden( + f"quality guard '{name}' in {DEFAULT_RUNNER} has no rule", + 'add a [[rule]] with enforcer.kind = "quality-guard"', + ) + ) + + # Ruff: format checks in CI, explicitly selected codes, per-file ignores. + ruff_rules = [r for r in rules if r.enforcer.kind is Kind.RUFF] + runs_format = any( + "ruff format" in job.run_text + for workflow in registry.discovery.workflows + for job in workflow_jobs(root / workflow)[0].values() + ) + if runs_format and not any(r.enforcer.get("mode") == "format" for r in ruff_rules): + violations.append( + _hidden( + "CI runs 'ruff format' but no rule describes it", + "add a ruff format rule", + ) + ) + lint = ruff_lint_config(root) + lint_codes = { + c + for r in ruff_rules + if r.enforcer.get("mode") == "lint" + for c in r.enforcer.get("codes", []) + } + for code in list(lint.get("select", [])) + list(lint.get("extend-select", [])): + if code not in lint_codes: + violations.append( + _hidden( + f"ruff selects '{code}' but no rule describes it", + "add the code to a ruff lint rule", + ) + ) + ignore_patterns = { + r.enforcer.get("pattern") + for r in ruff_rules + if r.enforcer.get("mode") == "per-file-ignore" + } + for pattern in lint.get("per-file-ignores", {}): + if pattern not in ignore_patterns: + violations.append( + _hidden( + f"ruff per-file-ignores '{pattern}' has no rule", + "add a ruff per-file-ignore rule explaining the exemption", + ) + ) + + # mypy scope. + covered = { + p + for r in rules + if r.enforcer.kind is Kind.MYPY + for p in r.enforcer.get("paths", []) + } + for path in mypy_files(root): + if path not in covered: + violations.append( + _hidden( + f"mypy checks '{path}' but no rule describes it", + "add it to a mypy rule's paths", + ) + ) + + # Guard-style tests. + registered_nodes = _registered_nodes(registry) + ignored_tests = { + split_node(i.target)[0] + "::" + split_node(i.target)[1] + for i in registry.discovery.ignore_tests + } + for node in sorted( + _discovered_tests(registry, root) - registered_nodes - ignored_tests + ): + violations.append( + _hidden( + f"guard test {node} is not referenced by any rule", + "add it to a pytest rule's nodes, or to discovery.ignore_tests with a reason", + ) + ) + + # CI jobs. + referenced_jobs = {(ref.workflow, ref.job) for r in rules for ref in r.ci} + ignored_jobs = {i.target for i in registry.discovery.ignore_jobs} + for workflow in registry.discovery.workflows: + for job_id in workflow_jobs(root / workflow)[0]: + if (workflow, job_id) not in referenced_jobs and job_id not in ignored_jobs: + violations.append( + _hidden( + f"CI job '{job_id}' in {workflow} enforces no registered rule", + "reference it from a rule's [[rule.ci]], or add it to discovery.ignore_jobs with a reason", + ) + ) + return violations diff --git a/scripts/rules/guard.py b/scripts/rules/guard.py new file mode 100644 index 000000000..f6c703e2c --- /dev/null +++ b/scripts/rules/guard.py @@ -0,0 +1,130 @@ +"""The registry guard: every check in one pass.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import date +from pathlib import Path + +from scripts.rules.baseline import ( + BaseUnavailable, + check_baselines, + default_base_ref, + load_base_registry, +) +from scripts.rules.discover import discover_hidden +from scripts.rules.load import load_registry +from scripts.rules.model import Violation +from scripts.rules.render import ( + AGENTS_MD, + RULES_MD, + extract_block, + render_agents_block, + render_rules_md, +) +from scripts.rules.resolve import ( + clear_caches, + resolve_ci, + resolve_enforcer, + resolve_proof_ci, +) +from scripts.rules.validate import validate + +REPO_ROOT = Path(__file__).resolve().parents[2] +RENDER_FIX = "run python -m scripts.rules render and commit the result" + + +@dataclass +class GuardResult: + violations: list[Violation] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + +def _stale_outputs(registry, root: Path) -> list[Violation]: + violations = [] + rules_md = root / RULES_MD + if not rules_md.exists(): + violations.append( + Violation(None, "render", f"{RULES_MD} is missing", RENDER_FIX) + ) + elif rules_md.read_text() != render_rules_md(registry): + violations.append(Violation(None, "render", f"{RULES_MD} is stale", RENDER_FIX)) + agents = root / AGENTS_MD + block = extract_block(agents.read_text()) if agents.exists() else None + if block is None: + violations.append( + Violation( + None, "render", f"{AGENTS_MD} has no generated rules block", RENDER_FIX + ) + ) + elif block != render_agents_block(registry): + violations.append( + Violation( + None, "render", f"the rules block in {AGENTS_MD} is stale", RENDER_FIX + ) + ) + return violations + + +def run_guard( + root: Path = REPO_ROOT, + *, + base_ref: str | None = None, + today: date | None = None, + compare_base: bool = True, +) -> GuardResult: + clear_caches() + result = GuardResult() + registry, schema_violations = load_registry(root) + result.violations += schema_violations + if registry is None: + return result + + result.violations += validate(registry, root, today or date.today()) + for rule in registry.rules: + result.violations += resolve_enforcer(rule, root) + result.violations += resolve_ci(rule, root) + result.violations += resolve_proof_ci(rule, root) + result.violations += discover_hidden(registry, root) + result.violations += _stale_outputs(registry, root) + + ref = base_ref or default_base_ref() + if not compare_base: + return result + if ref is None: + result.notes.append( + "no base ref (set --base-ref or GITHUB_BASE_REF); baseline comparison skipped" + ) + else: + try: + base = load_base_registry(root, ref) + except BaseUnavailable: + if os.environ.get("GITHUB_ACTIONS"): + result.violations.append( + Violation( + None, + "baseline", + f"base ref {ref} is not available", + "check out with fetch-depth: 0", + ) + ) + else: + result.notes.append( + f"base ref {ref} is not available; baseline comparison skipped" + ) + else: + if base is None: + result.notes.append( + f"{ref} has no registry yet; baseline comparison skipped" + ) + else: + violations, notes = check_baselines(registry, base) + result.violations += violations + result.notes += notes + return result + + +def check() -> list[str]: + """Adapter for scripts/run_quality_guards.py.""" + return [str(v) for v in run_guard(REPO_ROOT).violations] diff --git a/scripts/rules/load.py b/scripts/rules/load.py new file mode 100644 index 000000000..10b6faa60 --- /dev/null +++ b/scripts/rules/load.py @@ -0,0 +1,353 @@ +"""Load ``rules/registry.toml`` into the typed model. + +Parsing reports structural problems (wrong types, unknown keys, bad enum values) +as violations instead of raising, so one run shows every problem at once. +""" + +from __future__ import annotations + +import tomllib +from datetime import date +from pathlib import Path +from typing import Any + +from scripts.rules.model import ( + Baseline, + Category, + CiRef, + Discovery, + Enforcer, + Ignored, + Kind, + Proof, + Registry, + Rule, + RuleException, + Status, + Tier, + Violation, +) + +REGISTRY_PATH = Path("rules/registry.toml") + +_TOP_KEYS = {"schema_version", "repo", "tool_version", "category", "discovery", "rule"} +_RULE_KEYS = { + "id", + "category", + "statement", + "rationale", + "scope", + "tier", + "status", + "autofix", + "review_by", + "enforcer", + "ci", + "proof", + "baseline", + "exception", + "downgrade", +} +_DISCOVERY_KEYS = { + "workflows", + "guard_test_globs", + "extra_guard_tests", + "ignore_jobs", + "ignore_tests", +} + + +class _Collector: + def __init__(self) -> None: + self.violations: list[Violation] = [] + + def add(self, rule_id: str | None, message: str, fix: str) -> None: + self.violations.append(Violation(rule_id, "schema", message, fix)) + + def unknown_keys( + self, rule_id: str | None, table: dict[str, Any], allowed: set[str], where: str + ) -> None: + for key in sorted(set(table) - allowed): + self.add( + rule_id, + f"unknown key '{key}' in {where}", + f"remove it or use one of {sorted(allowed)}", + ) + + def string( + self, rule_id: str | None, table: dict[str, Any], key: str, where: str + ) -> str: + value = table.get(key) + if isinstance(value, str) and value.strip(): + return value + self.add(rule_id, f"{where}.{key} must be a non-empty string", f"set {key}") + return "" + + def string_list( + self, + rule_id: str | None, + table: dict[str, Any], + key: str, + where: str, + *, + required: bool = True, + ) -> tuple[str, ...]: + value = table.get(key) + if value is None and not required: + return () + if ( + isinstance(value, list) + and all(isinstance(item, str) and item for item in value) + and (value or not required) + ): + return tuple(value) + self.add( + rule_id, + f"{where}.{key} must be a non-empty list of strings", + f'set {key} = ["..."]', + ) + return () + + def enum(self, rule_id: str | None, enum_type: Any, value: Any, where: str) -> Any: + try: + return enum_type(value) + except ValueError: + allowed = [member.value for member in enum_type] + self.add(rule_id, f"{where} is {value!r}", f"use one of {allowed}") + return None + + def day(self, rule_id: str | None, value: Any, where: str) -> date | None: + if isinstance(value, date): + return value + if isinstance(value, str): + try: + return date.fromisoformat(value) + except ValueError: + pass + self.add(rule_id, f"{where} must be a date", "write it as YYYY-MM-DD") + return None + + +def _ignored(c: _Collector, raw: Any, key: str, field: str) -> tuple[Ignored, ...]: + if raw is None: + return () + items = [] + for entry in raw if isinstance(raw, list) else [raw]: + if ( + isinstance(entry, dict) + and isinstance(entry.get(field), str) + and isinstance(entry.get("reason"), str) + and entry["reason"].strip() + ): + items.append(Ignored(entry[field], entry["reason"])) + else: + c.add( + None, + f"discovery.{key} entries need '{field}' and a non-empty 'reason'", + f'write {{ {field} = "...", reason = "..." }}', + ) + return tuple(items) + + +def _discovery(c: _Collector, raw: Any) -> Discovery: + if raw is None: + return Discovery() + if not isinstance(raw, dict): + c.add(None, "discovery must be a table", "write [discovery]") + return Discovery() + c.unknown_keys(None, raw, _DISCOVERY_KEYS, "discovery") + workflows = c.string_list(None, raw, "workflows", "discovery", required=False) + return Discovery( + workflows=workflows or Discovery().workflows, + guard_test_globs=c.string_list( + None, raw, "guard_test_globs", "discovery", required=False + ), + extra_guard_tests=c.string_list( + None, raw, "extra_guard_tests", "discovery", required=False + ), + ignore_jobs=_ignored(c, raw.get("ignore_jobs"), "ignore_jobs", "job"), + ignore_tests=_ignored(c, raw.get("ignore_tests"), "ignore_tests", "node"), + ) + + +def _rule(c: _Collector, raw: Any, index: int) -> Rule | None: + if not isinstance(raw, dict): + c.add(None, f"rule #{index + 1} must be a table", "write [[rule]]") + return None + rule_id = raw.get("id") if isinstance(raw.get("id"), str) else None + label = rule_id or f"rule #{index + 1}" + c.unknown_keys(rule_id, raw, _RULE_KEYS, label) + + enforcer_raw = raw.get("enforcer") + enforcer = Enforcer(Kind.NONE) + if isinstance(enforcer_raw, dict): + kind = c.enum(rule_id, Kind, enforcer_raw.get("kind"), f"{label}.enforcer.kind") + fields = {k: v for k, v in enforcer_raw.items() if k != "kind"} + enforcer = Enforcer(kind or Kind.NONE, fields) + else: + c.add(rule_id, f"{label}.enforcer must be a table", "add [rule.enforcer]") + + ci_refs = [] + ci_raw = raw.get("ci", []) + for entry in ci_raw if isinstance(ci_raw, list) else [ci_raw]: + if not isinstance(entry, dict): + c.add(rule_id, f"{label}.ci entries must be tables", "write [[rule.ci]]") + continue + c.unknown_keys(rule_id, entry, {"workflow", "job", "invocation"}, f"{label}.ci") + ci_refs.append( + CiRef( + c.string(rule_id, entry, "workflow", f"{label}.ci"), + c.string(rule_id, entry, "job", f"{label}.ci"), + c.string(rule_id, entry, "invocation", f"{label}.ci"), + ) + ) + + proof = None + if "proof" in raw: + if isinstance(raw["proof"], dict): + proof = Proof(dict(raw["proof"])) + else: + c.add(rule_id, f"{label}.proof must be a table", "write [rule.proof]") + + baseline = None + if "baseline" in raw: + braw = raw["baseline"] + if isinstance(braw, dict) and isinstance(braw.get("count"), int): + c.unknown_keys(rule_id, braw, {"file", "count"}, f"{label}.baseline") + baseline = Baseline( + c.string(rule_id, braw, "file", f"{label}.baseline"), braw["count"] + ) + else: + c.add( + rule_id, + f"{label}.baseline needs 'file' and an integer 'count'", + "write [rule.baseline] with file and count", + ) + + exceptions = [] + exc_raw = raw.get("exception", []) + for entry in exc_raw if isinstance(exc_raw, list) else [exc_raw]: + if not isinstance(entry, dict): + c.add( + rule_id, + f"{label}.exception entries must be tables", + "write [[rule.exception]]", + ) + continue + c.unknown_keys( + rule_id, entry, {"path", "reason", "expires"}, f"{label}.exception" + ) + expires = c.day(rule_id, entry.get("expires"), f"{label}.exception.expires") + exceptions.append( + RuleException( + c.string(rule_id, entry, "path", f"{label}.exception"), + c.string(rule_id, entry, "reason", f"{label}.exception"), + expires or date.max, + ) + ) + + downgrade_reason = None + if "downgrade" in raw: + draw = raw["downgrade"] + if isinstance(draw, dict) and isinstance(draw.get("reason"), str): + downgrade_reason = draw["reason"] + else: + c.add( + rule_id, + f"{label}.downgrade needs a 'reason'", + 'write [rule.downgrade] reason = "..."', + ) + + autofix = raw.get("autofix") + if not isinstance(autofix, bool): + c.add(rule_id, f"{label}.autofix must be true or false", "set autofix") + autofix = False + + review_by = None + if "review_by" in raw: + review_by = c.day(rule_id, raw["review_by"], f"{label}.review_by") + + tier = c.enum(rule_id, Tier, raw.get("tier"), f"{label}.tier") + status = c.enum(rule_id, Status, raw.get("status"), f"{label}.status") + if tier is None or status is None: + return None + + return Rule( + id=c.string(rule_id, raw, "id", label), + category=c.string(rule_id, raw, "category", label), + statement=c.string(rule_id, raw, "statement", label), + rationale=c.string(rule_id, raw, "rationale", label), + scope=c.string_list(rule_id, raw, "scope", label), + tier=tier, + status=status, + autofix=autofix, + enforcer=enforcer, + ci=tuple(ci_refs), + review_by=review_by, + proof=proof, + baseline=baseline, + exceptions=tuple(exceptions), + downgrade_reason=downgrade_reason, + ) + + +def parse(raw: dict[str, Any]) -> tuple[Registry | None, list[Violation]]: + """Turn raw TOML data into a ``Registry``, collecting every schema problem.""" + c = _Collector() + c.unknown_keys(None, raw, _TOP_KEYS, "registry") + + schema_version = raw.get("schema_version") + if schema_version != 1: + c.add(None, f"schema_version is {schema_version!r}", "set schema_version = 1") + + categories = [] + for entry in raw.get("category", []): + if isinstance(entry, dict) and isinstance(entry.get("id"), str): + c.unknown_keys(None, entry, {"id", "title"}, f"category {entry['id']}") + categories.append( + Category(entry["id"], c.string(None, entry, "title", "category")) + ) + else: + c.add( + None, + "each [[category]] needs an id and title", + "fix the category table", + ) + + rules = [_rule(c, entry, i) for i, entry in enumerate(raw.get("rule", []))] + + registry = Registry( + schema_version=1, + repo=c.string(None, raw, "repo", "registry"), + tool_version=c.string(None, raw, "tool_version", "registry"), + categories=tuple(categories), + discovery=_discovery(c, raw.get("discovery")), + rules=tuple(rule for rule in rules if rule is not None), + ) + return registry, c.violations + + +def load_text(text: str) -> tuple[Registry | None, list[Violation]]: + try: + raw = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + return None, [ + Violation( + None, + "schema", + f"registry is not valid TOML ({error})", + "fix the TOML syntax", + ) + ] + return parse(raw) + + +def load_registry(root: Path) -> tuple[Registry | None, list[Violation]]: + path = root / REGISTRY_PATH + if not path.exists(): + return None, [ + Violation( + None, "schema", f"{REGISTRY_PATH} is missing", f"create {REGISTRY_PATH}" + ) + ] + return load_text(path.read_text()) diff --git a/scripts/rules/model.py b/scripts/rules/model.py new file mode 100644 index 000000000..38247b9d4 --- /dev/null +++ b/scripts/rules/model.py @@ -0,0 +1,192 @@ +"""Data model for the rules registry.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import date +from enum import StrEnum +from typing import Any + + +class Tier(StrEnum): + MECHANICAL = "mechanical" + METRIC = "metric" + JUDGMENT = "judgment" + + +class Status(StrEnum): + ENFORCED = "enforced" + RATCHETED = "ratcheted" + ADVISORY = "advisory" + DOC_ONLY = "doc-only" + + +class Kind(StrEnum): + PYTEST = "pytest" + QUALITY_GUARD = "quality-guard" + RUFF = "ruff" + MYPY = "mypy" + PYRIGHT = "pyright" + IMPORT_LINTER = "import-linter" + WORKFLOW_JOB = "workflow-job" + DOCS_GUARD = "docs-guard" + LLM_REVIEW = "llm-review" + NONE = "none" + + +@dataclass(frozen=True) +class Violation: + """One guard failure. ``fix`` always tells the reader what to do next.""" + + rule_id: str | None + check: str + message: str + fix: str + + def __str__(self) -> str: + subject = self.rule_id or "registry" + return f"[{subject}] {self.check}: {self.message}. Fix: {self.fix}." + + +@dataclass(frozen=True) +class Category: + id: str + title: str + + +@dataclass(frozen=True) +class Ignored: + """A discovered item deliberately left out of the registry.""" + + target: str + reason: str + + +@dataclass(frozen=True) +class Discovery: + workflows: tuple[str, ...] = (".github/workflows/pr.yml",) + guard_test_globs: tuple[str, ...] = () + extra_guard_tests: tuple[str, ...] = () + ignore_jobs: tuple[Ignored, ...] = () + ignore_tests: tuple[Ignored, ...] = () + + +@dataclass(frozen=True) +class Enforcer: + kind: Kind + fields: Mapping[str, Any] = field(default_factory=dict) + + def get(self, key: str, default: Any = None) -> Any: + return self.fields.get(key, default) + + @property + def description(self) -> str: + """Short human-readable description of what enforces the rule.""" + kind, fields = self.kind, self.fields + if kind is Kind.PYTEST: + prefix = f"{fields['project'].rstrip('/')}/" if "project" in fields else "" + files = sorted({node.split("::", 1)[0] for node in fields.get("nodes", [])}) + return "pytest " + ", ".join(f"`{prefix}{path}`" for path in files) + if kind is Kind.QUALITY_GUARD: + return f"quality guard `{fields.get('name', '')}`" + if kind is Kind.RUFF: + codes = ", ".join(fields.get("codes", [])) + mode = fields.get("mode", "") + if mode == "format": + return "ruff format" + if mode == "per-file-ignore": + return ( + f"ruff per-file ignore of {codes} for `{fields.get('pattern', '')}`" + ) + return f"ruff check ({codes})" + if kind is Kind.MYPY: + return "mypy" + if kind is Kind.WORKFLOW_JOB: + return "CI job" + if kind in (Kind.PYRIGHT, Kind.IMPORT_LINTER): + return f"{kind.value} `{fields.get('config', '')}`" + if kind is Kind.DOCS_GUARD: + return f"docs guard on `{fields.get('doc', '')}`" + if kind is Kind.LLM_REVIEW: + return f"LLM review with `{fields.get('prompt', '')}`" + return "nothing" + + @property + def expects_proof(self) -> bool: + """Custom checkers need proof; third-party tools and CI jobs are the check.""" + return self.kind in ( + Kind.PYTEST, + Kind.QUALITY_GUARD, + Kind.DOCS_GUARD, + Kind.IMPORT_LINTER, + Kind.LLM_REVIEW, + ) + + +@dataclass(frozen=True) +class CiRef: + workflow: str + job: str + invocation: str + + +@dataclass(frozen=True) +class Proof: + fields: Mapping[str, Any] + + +@dataclass(frozen=True) +class Baseline: + file: str + count: int + + +@dataclass(frozen=True) +class RuleException: + path: str + reason: str + expires: date + + +@dataclass(frozen=True) +class Rule: + id: str + category: str + statement: str + rationale: str + scope: tuple[str, ...] + tier: Tier + status: Status + autofix: bool + enforcer: Enforcer + ci: tuple[CiRef, ...] = () + review_by: date | None = None + proof: Proof | None = None + baseline: Baseline | None = None + exceptions: tuple[RuleException, ...] = () + downgrade_reason: str | None = None + + @property + def blocking(self) -> bool: + """Derived, never declared: whether a violation can fail a merge.""" + return ( + self.status in (Status.ENFORCED, Status.RATCHETED) + and self.tier is not Tier.JUDGMENT + ) + + +@dataclass(frozen=True) +class Registry: + schema_version: int + repo: str + tool_version: str + categories: tuple[Category, ...] + discovery: Discovery + rules: tuple[Rule, ...] + + def rule(self, rule_id: str) -> Rule | None: + for rule in self.rules: + if rule.id == rule_id: + return rule + return None diff --git a/scripts/rules/render.py b/scripts/rules/render.py new file mode 100644 index 000000000..9458349b2 --- /dev/null +++ b/scripts/rules/render.py @@ -0,0 +1,271 @@ +"""Render the registry into ``docs/generated/rules.md`` and the AGENTS.md block. + +Output is deterministic (fixed ordering, no timestamps, no dependence on today's +date) so the guard can regenerate it and compare. +""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path + +from scripts.rules import TOOL_VERSION +from scripts.rules.model import Kind, Registry, Rule, Status, Tier + +RULES_MD = Path("docs/generated/rules.md") +AGENTS_MD = Path("AGENTS.md") +SKILL_DOC = "docs/engineering/skills/rules-registry.md" +BEGIN = "" +END = "" +MERMAID_NODE_LIMIT = 40 +MERMAID_CHUNK = 15 +AGENTS_COLLAPSE_AT = 60 + + +def tool_hash() -> str: + digest = hashlib.sha256() + for path in sorted(Path(__file__).parent.glob("*.py")): + digest.update(path.name.encode()) + digest.update(path.read_bytes()) + return digest.hexdigest()[:12] + + +def _cell(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def _ordered(registry: Registry) -> list[tuple[str, str, list[Rule]]]: + """Categories in declared order, each with its rules sorted by id.""" + groups = [] + for category in registry.categories: + rules = sorted( + (r for r in registry.rules if r.category == category.id), key=lambda r: r.id + ) + if rules: + groups.append((category.id, category.title, rules)) + return groups + + +def _ci_label(rule: Rule) -> str: + return ", ".join(f"`{Path(ref.workflow).name}:{ref.job}`" for ref in rule.ci) or "—" + + +def _unproven(rule: Rule) -> bool: + return rule.blocking and rule.proof is None and rule.enforcer.expects_proof + + +def _proof_label(rule: Rule) -> str: + if rule.proof is not None: + return "yes" + return "**unproven**" if _unproven(rule) else "—" + + +def _enforced_by(rule: Rule) -> str: + """What enforces a rule, naming the CI job when the job itself is the check.""" + if rule.enforcer.kind is Kind.WORKFLOW_JOB: + jobs = ", ".join(f"`{ref.job}`" for ref in rule.ci) + return f"CI job {jobs}" + return rule.enforcer.description + + +def _mermaid_id(prefix: str, text: str) -> str: + return prefix + "_" + re.sub(r"[^A-Za-z0-9]", "_", text) + + +def _mermaid(rules: list[Rule]) -> list[str]: + lines = ["```mermaid", "flowchart LR"] + declared: set[str] = set() + edges: list[str] = [] + + def node(node_id: str, label: str, shape: str = "box") -> str: + if node_id not in declared: + declared.add(node_id) + safe = label.replace('"', "'") + if shape == "round": + lines.append(f' {node_id}("{safe}")') + elif shape == "flag": + lines.append(f' {node_id}>"{safe}"]') + else: + lines.append(f' {node_id}["{safe}"]') + return node_id + + for rule in rules: + rule_node = node( + _mermaid_id("r", rule.id), f"{rule.id} ({rule.status})", "round" + ) + if rule.enforcer.kind is Kind.NONE: + gap = node("gap_unenforced", "not enforced", "flag") + edges.append(f" {rule_node} -.-> {gap}") + continue + label = _enforced_by(rule).replace("`", "") + enforcer_node = node(_mermaid_id("e", label), label) + edges.append(f" {rule_node} --> {enforcer_node}") + for ref in rule.ci: + job_label = f"CI {Path(ref.workflow).name}:{ref.job}" + job_node = node(_mermaid_id("j", job_label), job_label) + edge = f" {enforcer_node} --> {job_node}" + if edge not in edges: + edges.append(edge) + if not rule.ci: + local = node("local_only", "not run in CI", "flag") + edge = f" {enforcer_node} -.-> {local}" + if edge not in edges: + edges.append(edge) + return lines + edges + ["```"] + + +def _mermaid_chunks(rules: list[Rule]) -> list[list[str]]: + graph = _mermaid(rules) + node_count = sum(1 for line in graph if re.match(r"^ \w+[\[(>]", line)) + if node_count <= MERMAID_NODE_LIMIT: + return [graph] + return [ + _mermaid(rules[i : i + MERMAID_CHUNK]) + for i in range(0, len(rules), MERMAID_CHUNK) + ] + + +def _bullets(items: list[str]) -> list[str]: + return items or ["- None."] + + +def render_rules_md(registry: Registry) -> str: + rules = sorted(registry.rules, key=lambda r: r.id) + out = [ + "# Rules", + "", + f"Generated by `python -m scripts.rules render` (scripts/rules v{TOOL_VERSION}, " + f"tool sha256 `{tool_hash()}`) from `rules/registry.toml`. Do not edit by hand.", + "", + f"Every rule that governs `{registry.repo}` is listed here with how it is enforced. " + "Blocking rules fail CI. The Gaps section lists what is written down but not yet " + f"enforced or proven. To add or change a rule, see `{SKILL_DOC}`.", + "", + "## Summary", + "", + "| Tier | " + " | ".join(s.value for s in Status) + " | Total |", + "| --- | " + " | ".join("---:" for _ in Status) + " | ---: |", + ] + for tier in Tier: + counts = [ + sum(1 for r in rules if r.tier is tier and r.status is s) for s in Status + ] + out.append( + f"| {tier.value} | " + + " | ".join(str(c) for c in counts) + + f" | {sum(counts)} |" + ) + blocking = sum(1 for r in rules if r.blocking) + out += ["", f"**Blocking:** {blocking} of {len(rules)} rules.", "", "## Gaps", ""] + + def section(title: str, items: list[str]) -> None: + out.extend([f"### {title}", "", *_bullets(items), ""]) + + section( + "Not enforced (doc-only)", + [f"- `{r.id}`: {r.statement}" for r in rules if r.status is Status.DOC_ONLY], + ) + section( + "Advisory (checked but not blocking)", + [f"- `{r.id}`: {r.statement}" for r in rules if r.status is Status.ADVISORY], + ) + section( + "Unproven (no proof that the checker rejects a bad example)", + [f"- `{r.id}`" for r in rules if _unproven(r)], + ) + section( + "Ratcheted (existing violations still tolerated)", + [ + f"- `{r.id}`: {r.baseline.count} remaining in `{r.baseline.file}`" + for r in rules + if r.baseline + ], + ) + dated = [f"- `{r.id}`: review by {r.review_by}" for r in rules if r.review_by] + [ + f"- `{r.id}`: exception for `{e.path}` until {e.expires} ({e.reason})" + for r in rules + for e in r.exceptions + ] + section("Exceptions and review dates", sorted(dated)) + + for _, title, group in _ordered(registry): + out += [ + f"## {title}", + "", + "| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + for r in group: + out.append( + f"| `{r.id}` | {_cell(r.statement)} | {r.tier} | {r.status} | " + f"{'yes' if r.blocking else 'no'} | {_cell(_enforced_by(r))} | " + f"{_ci_label(r)} | {_proof_label(r)} |" + ) + out.append("") + for chunk in _mermaid_chunks(group): + out += chunk + [""] + return "\n".join(out).rstrip() + "\n" + + +def _agents_line(rule: Rule) -> str: + if rule.status is Status.DOC_ONLY: + how = "doc-only, not checked: follow it yourself" + elif rule.status is Status.ADVISORY: + how = f"advisory: {_enforced_by(rule)}, not blocking" + elif rule.blocking: + how = f"blocking: {_enforced_by(rule)}" + else: + how = f"{rule.status}: {_enforced_by(rule)}" + return f"- `{rule.id}`: {rule.statement} ({how})" + + +def render_agents_block(registry: Registry) -> str: + out = [ + BEGIN, + "", + "Every rule for this repository lives in `rules/registry.toml` and is rendered,", + "with its enforcement and gaps, in `docs/generated/rules.md`. Blocking rules", + "fail CI. Doc-only rules are not checked automatically, so follow them yourself.", + f"To add or change a rule, read `{SKILL_DOC}`.", + "", + ] + collapse = len(registry.rules) > AGENTS_COLLAPSE_AT + for _, title, group in _ordered(registry): + out.append(f"### {title}") + out.append("") + if collapse: + enforced = [f"`{r.id}`" for r in group if r.status is not Status.DOC_ONLY] + if enforced: + out.append("- Checked: " + ", ".join(enforced)) + out += [_agents_line(r) for r in group if r.status is Status.DOC_ONLY] + else: + out += [_agents_line(r) for r in group] + out.append("") + out.append(END) + return "\n".join(out) + + +def splice_block(text: str, block: str) -> str: + """Replace the generated block in AGENTS.md, appending a Rules section if absent.""" + start, end = text.find(BEGIN), text.find(END) + if start != -1 and end != -1 and end > start: + return text[:start] + block + text[end + len(END) :] + return text.rstrip() + "\n\n## Rules\n\n" + block + "\n" + + +def extract_block(text: str) -> str | None: + start, end = text.find(BEGIN), text.find(END) + if start == -1 or end == -1 or end < start: + return None + return text[start : end + len(END)] + + +def write_outputs(registry: Registry, root: Path) -> list[Path]: + rules_md = root / RULES_MD + rules_md.parent.mkdir(parents=True, exist_ok=True) + rules_md.write_text(render_rules_md(registry)) + agents = root / AGENTS_MD + current = agents.read_text() if agents.exists() else "# Agent Instructions\n" + agents.write_text(splice_block(current, render_agents_block(registry))) + return [rules_md, agents] diff --git a/scripts/rules/resolve.py b/scripts/rules/resolve.py new file mode 100644 index 000000000..1e47b09c7 --- /dev/null +++ b/scripts/rules/resolve.py @@ -0,0 +1,471 @@ +"""Check that each rule's enforcer and CI wiring really exist. + +Everything here reads files statically (``ast``, ``tomllib`` and the workflow +subset reader) so it needs no project dependencies and never imports project +code. +""" + +from __future__ import annotations + +import ast +import re +import tomllib +from collections.abc import Callable +from functools import cache +from pathlib import Path +from typing import Any + +from scripts.rules.model import CiRef, Kind, Rule, Violation +from scripts.rules.workflows import JobBlock, parse_jobs + +DEFAULT_RUNNER = "scripts/run_quality_guards.py" +RUFF_DEFAULT_SELECT = ("E4", "E7", "E9", "F") +PROOF_CI_FIELDS = ("workflow", "job", "invocation") + + +# --- shared readers ------------------------------------------------------- + + +@cache +def _read(path: Path) -> str | None: + return path.read_text() if path.is_file() else None + + +def clear_caches() -> None: + _read.cache_clear() + workflow_jobs.cache_clear() + test_names.cache_clear() + + +@cache +def workflow_jobs(path: Path) -> tuple[dict[str, JobBlock], tuple[str, ...]]: + text = _read(path) + if text is None: + return {}, (f"{path.name} is missing",) + jobs, problems = parse_jobs(text) + return jobs, tuple(problems) + + +@cache +def test_names(path: Path) -> frozenset[str] | None: + """Pytest-collectable names in a file: ``func`` and ``Class::method``.""" + text = _read(path) + if text is None: + return None + names = set() + for node in ast.parse(text).body: + if isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and node.name.startswith("test"): + names.add(node.name) + elif isinstance(node, ast.ClassDef) and node.name.startswith("Test"): + for item in node.body: + if isinstance( + item, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and item.name.startswith("test"): + names.add(f"{node.name}::{item.name}") + return frozenset(names) + + +def split_node(node: str) -> tuple[str, str]: + """``path::name[param]`` -> (``path``, ``name``).""" + path, _, name = node.partition("::") + return path, re.sub(r"\[.*\]$", "", name) + + +def root_relative_nodes(rule: Rule) -> list[str]: + """Pytest nodes of a rule, prefixed with the project directory if any.""" + project = rule.enforcer.get("project") + nodes = rule.enforcer.get("nodes", []) + prefix = f"{project.rstrip('/')}/" if project else "" + return [f"{prefix}{node}" for node in nodes] + + +def node_exists(root: Path, node: str) -> str | None: + """Return a problem description, or None if the node resolves.""" + path, name = split_node(node) + if not name: + return f"'{node}' has no '::test_name' part" + names = test_names(root / path) + if names is None: + return f"test file {path} does not exist" + if name not in names: + return f"{path} has no test '{name}'" + return None + + +def guard_names(root: Path, runner: str = DEFAULT_RUNNER) -> list[str] | None: + """String names in the ``GUARDS`` tuple of the quality-guard runner.""" + text = _read(root / runner) + if text is None: + return None + for node in ast.parse(text).body: + target = None + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target, value = node.targets[0], node.value + elif isinstance(node, ast.AnnAssign): + target, value = node.target, node.value + if ( + isinstance(target, ast.Name) + and target.id == "GUARDS" + and isinstance(value, ast.Tuple) + ): + names = [] + for element in value.elts: + if ( + isinstance(element, ast.Tuple) + and element.elts + and isinstance(element.elts[0], ast.Constant) + and isinstance(element.elts[0].value, str) + ): + names.append(element.elts[0].value) + return names + return [] + + +def _toml(path: Path) -> dict[str, Any]: + text = _read(path) + return tomllib.loads(text) if text else {} + + +def ruff_lint_config(root: Path) -> dict[str, Any]: + """The ``lint`` table from ruff.toml, or ``[tool.ruff.lint]`` in pyproject.""" + ruff_toml = _toml(root / "ruff.toml") + if ruff_toml: + return ruff_toml.get("lint", {}) + return ( + _toml(root / "pyproject.toml").get("tool", {}).get("ruff", {}).get("lint", {}) + ) + + +def ruff_selected(root: Path) -> tuple[str, ...]: + lint = ruff_lint_config(root) + selected = tuple(lint.get("select", RUFF_DEFAULT_SELECT)) + return selected + tuple(lint.get("extend-select", ())) + + +def mypy_files(root: Path) -> list[str]: + files = ( + _toml(root / "pyproject.toml").get("tool", {}).get("mypy", {}).get("files", []) + ) + return [files] if isinstance(files, str) else list(files) + + +def make_target_paths(root: Path, target: str) -> list[str] | None: + """Pytest path arguments in a Makefile target's recipe.""" + text = _read(root / "Makefile") + if text is None: + return None + lines = text.splitlines() + for index, line in enumerate(lines): + if re.match(rf"^{re.escape(target)}\s*:", line): + recipe = [] + for body in lines[index + 1 :]: + if not body.startswith("\t"): + break + recipe.append(body) + paths = [] + for body in recipe: + if "pytest" not in body: + continue + tokens = body.split("pytest", 1)[1].split() + paths += [ + t + for t in tokens + if not t.startswith("-") and ("/" in t or t.endswith(".py")) + ] + return paths + return None + + +# --- per-kind resolvers --------------------------------------------------- + + +def _v(rule: Rule, message: str, fix: str) -> Violation: + return Violation(rule.id, "enforcer", message, fix) + + +def _resolve_pytest(rule: Rule, root: Path) -> list[Violation]: + violations = [] + project = rule.enforcer.get("project") + if project and not (root / project).is_dir(): + violations.append( + _v( + rule, + f"project directory {project} does not exist", + "fix enforcer.project", + ) + ) + for node in root_relative_nodes(rule): + problem = node_exists(root, node) + if problem: + violations.append( + _v( + rule, + problem, + "point enforcer.nodes at an existing test, or restore the test", + ) + ) + return violations + + +def _resolve_quality_guard(rule: Rule, root: Path) -> list[Violation]: + runner = rule.enforcer.get("runner", DEFAULT_RUNNER) + names = guard_names(root, runner) + if names is None: + return [ + _v(rule, f"guard runner {runner} does not exist", "fix enforcer.runner") + ] + name = rule.enforcer.get("name") + if name not in names: + return [ + _v( + rule, + f"no guard named '{name}' in GUARDS of {runner}", + f"use one of {names}", + ) + ] + return [] + + +def _resolve_ruff(rule: Rule, root: Path) -> list[Violation]: + mode = rule.enforcer.get("mode") + codes = rule.enforcer.get("codes", []) + if mode == "lint": + selected = ruff_selected(root) + missing = [ + code for code in codes if not any(code.startswith(s) for s in selected) + ] + if missing: + return [ + _v( + rule, + f"ruff does not select {missing} (selected: {list(selected)})", + "add them to the ruff select list or fix enforcer.codes", + ) + ] + elif mode == "per-file-ignore": + ignores = ruff_lint_config(root).get("per-file-ignores", {}) + pattern = rule.enforcer.get("pattern") + if sorted(ignores.get(pattern, [])) != sorted(codes): + return [ + _v( + rule, + f"ruff per-file-ignores has no entry {pattern!r} = {codes}", + "match enforcer.pattern/codes to the ruff config", + ) + ] + return [] + + +def _resolve_mypy(rule: Rule, root: Path) -> list[Violation]: + configured = mypy_files(root) + missing = [p for p in rule.enforcer.get("paths", []) if p not in configured] + if missing: + return [ + _v( + rule, + f"{missing} not in [tool.mypy].files", + "add them to [tool.mypy].files or fix enforcer.paths", + ) + ] + return [] + + +def _resolve_file_field(field_name: str) -> Callable[[Rule, Path], list[Violation]]: + def resolve(rule: Rule, root: Path) -> list[Violation]: + value = rule.enforcer.get(field_name) + if value and not (root / value).exists(): + return [ + _v( + rule, + f"{field_name} file {value} does not exist", + f"create it or fix enforcer.{field_name}", + ) + ] + return [] + + return resolve + + +def _resolve_workflow_job(rule: Rule, root: Path) -> list[Violation]: + step = rule.enforcer.get("step") + if not step: + return [] + violations = [] + for ref in rule.ci: + job = workflow_jobs(root / ref.workflow)[0].get(ref.job) + if job and not any(step in name for name in job.step_names): + violations.append( + _v( + rule, + f"job '{ref.job}' has no step named like '{step}'", + "fix enforcer.step", + ) + ) + return violations + + +RESOLVERS: dict[Kind, Callable[[Rule, Path], list[Violation]]] = { + Kind.PYTEST: _resolve_pytest, + Kind.QUALITY_GUARD: _resolve_quality_guard, + Kind.RUFF: _resolve_ruff, + Kind.MYPY: _resolve_mypy, + Kind.PYRIGHT: _resolve_file_field("config"), + Kind.IMPORT_LINTER: _resolve_file_field("config"), + Kind.WORKFLOW_JOB: _resolve_workflow_job, + Kind.DOCS_GUARD: _resolve_file_field("doc"), + Kind.LLM_REVIEW: _resolve_file_field("prompt"), + Kind.NONE: lambda rule, root: [], +} + + +def resolve_enforcer(rule: Rule, root: Path) -> list[Violation]: + return RESOLVERS[rule.enforcer.kind](rule, root) + + +# --- CI wiring ------------------------------------------------------------ + + +def _job_for( + rule_id: str, ref: CiRef, root: Path, check: str, field: str +) -> tuple[JobBlock | None, list[Violation]]: + """The job a CI reference names, if it exists and runs the invocation.""" + + def v(message: str, fix: str) -> Violation: + return Violation(rule_id, check, message, fix) + + if not (root / ref.workflow).is_file(): + return None, [ + v(f"workflow {ref.workflow} does not exist", f"fix {field}.workflow") + ] + jobs, problems = workflow_jobs(root / ref.workflow) + violations = [ + v( + f"{ref.workflow}: {problem}", + "keep workflows within the supported YAML subset", + ) + for problem in problems + ] + job = jobs.get(ref.job) + if job is None: + violations.append( + v(f"no job '{ref.job}' in {ref.workflow}", f"use one of {sorted(jobs)}") + ) + return None, violations + if not job.invokes(ref.invocation): + violations.append( + v( + f"job '{ref.job}' in {ref.workflow} does not run '{ref.invocation}'", + f"correct {field}.invocation, or make the job run it", + ) + ) + return None, violations + return job, violations + + +def resolve_ci(rule: Rule, root: Path) -> list[Violation]: + """Each CI reference names a real job that really runs the enforcer.""" + violations = [] + for ref in rule.ci: + job, problems = _job_for(rule.id, ref, root, "ci", "rule.ci") + violations += problems + if job is not None and rule.enforcer.kind is Kind.PYTEST: + violations += _test_coverage( + rule.id, + root_relative_nodes(rule), + rule.enforcer.get("project"), + ref.invocation, + job, + root, + "ci", + ) + return violations + + +def resolve_proof_ci(rule: Rule, root: Path) -> list[Violation]: + """A proof test must run in CI, or nothing shows the checker still works. + + The proof test itself asserts that the bad fixture is rejected and the good + one accepted; the guard checks that some CI job collects it. A proof runs + in the rule's own CI job when the rule is pytest-enforced; otherwise the + proof names its job with ``workflow``, ``job`` and ``invocation``. + """ + if rule.proof is None or not isinstance(rule.proof.fields.get("test"), str): + return [] + fields = rule.proof.fields + project = fields.get("project") if isinstance(fields.get("project"), str) else None + prefix = f"{project.rstrip('/')}/" if project else "" + node = prefix + fields["test"] + if all(isinstance(fields.get(key), str) for key in PROOF_CI_FIELDS): + refs = [CiRef(fields["workflow"], fields["job"], fields["invocation"])] + elif rule.enforcer.kind is Kind.PYTEST: + refs = list(rule.ci) + else: + refs = [] + if not refs: + return [ + Violation( + rule.id, + "proof", + "the proof test is not tied to a CI job", + "add proof.workflow, proof.job and proof.invocation naming the job that runs it", + ) + ] + violations = [] + for ref in refs: + job, problems = _job_for(rule.id, ref, root, "proof", "proof") + violations += problems + if job is not None: + violations += _test_coverage( + rule.id, [node], project, ref.invocation, job, root, "proof" + ) + return violations + + +def _test_coverage( + rule_id: str, + nodes: list[str], + project: str | None, + invocation: str, + job: JobBlock, + root: Path, + check: str, +) -> list[Violation]: + """The job's test command actually collects these root-relative test nodes.""" + violations = [] + collected: list[str] | None = None + described = "" + if invocation.startswith("make "): + target = invocation.split()[1] + collected = make_target_paths(root, target) + described = f"'make {target}'" + elif "unittest discover" in invocation: + match = re.search(r"(?:-s|--start-directory)\s+(\S+)", invocation) + collected = [match.group(1)] if match else None + described = f"'{invocation}'" + if collected: + for node in nodes: + path = split_node(node)[0] + if not any( + path == p or path.startswith(p.rstrip("/") + "/") for p in collected + ): + violations.append( + Violation( + rule_id, + check, + f"{path} is not collected by {described} (collects {collected})", + "add it to what the job runs, or move the test", + ) + ) + if project and job.matrix: + if not any(project in values for values in job.matrix.values()): + violations.append( + Violation( + rule_id, + check, + f"project {project} is not in the matrix of job '{job.id}'", + "add the project to the job matrix", + ) + ) + return violations diff --git a/scripts/rules/tests/__init__.py b/scripts/rules/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/rules/tests/fixtures/policyengine_api_pr.yml b/scripts/rules/tests/fixtures/policyengine_api_pr.yml new file mode 100644 index 000000000..01d1cbbf8 --- /dev/null +++ b/scripts/rules/tests/fixtures/policyengine_api_pr.yml @@ -0,0 +1,201 @@ +name: Pull request + +on: pull_request + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + OLD_SIMULATION_GATEWAY_URL: ${{ vars.OLD_SIMULATION_GATEWAY_URL }} + +jobs: + ensure-policyengine-bundle-supported-by-simulation-api: + name: Ensure PolicyEngine bundle is supported by simulation API + runs-on: ubuntu-latest + environment: staging + env: + SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} + SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} + permissions: + contents: read + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install jq + run: sudo apt-get install -y jq + - name: Set up uv for installed bundle validation + uses: astral-sh/setup-uv@v6 + - name: Check simulation API supports updated PolicyEngine bundle + run: bash .github/check-policyengine-bundle-supported.sh --if-changed-from-base "${{ github.base_ref }}" + + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + - name: Install ruff + run: pip install ruff>=0.9.0 + - name: Format check with ruff + run: ruff format --check . + typecheck-v2: + name: Type-check API v2 + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v6 + - name: Install locked dependencies + run: uv sync --frozen --extra dev + - name: Type-check API v2 + run: uv run --frozen --extra dev mypy + quality-guards: + name: Quality guards + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + # The rules registry guard compares baselines with the base branch. + fetch-depth: 0 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Test the rules registry tool + run: python -m unittest discover -s scripts/rules/tests -t . + - name: Run quality guards + run: python scripts/run_quality_guards.py + + alembic-v1-check: + name: Alembic v1 qualification + uses: ./.github/workflows/alembic-v1-check.yml + alembic-v2-check: + name: Alembic v2 qualification + uses: ./.github/workflows/alembic-v2-check.yml + v2-integration-check: + name: Cross-database integration tests + uses: ./.github/workflows/v2-integration-check.yml + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + check-changelog: + name: Check changelog fragment + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check for changelog fragment + run: bash .github/scripts/check_changelog_fragment.sh + test_container_builds: + name: Docker + runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api + permissions: + contents: read + packages: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build container + run: docker build -t ghcr.io/policyengine/policyengine docker + test_cloud_run_container_builds: + name: Cloud Run container + runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api + permissions: + contents: read + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Build Cloud Run container + run: docker build -f gcp/cloud_run/Dockerfile -t policyengine-api-cloud-run:test . + test_env_vars: + name: Test environment variables + runs-on: ubuntu-latest + needs: ensure-policyengine-bundle-supported-by-simulation-api + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Wait until PolicyEngine bundle version is available on PyPI + run: .github/wait-for-pypi.sh + - name: Install dependencies + run: make install + - name: Run environment variable tests + run: pytest tests/env_variables/test_environment_variables.py + env: + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + # Compatibility name required by PolicyEngine Core. The GitHub + # credential name describes its provider, data, and read-only scope. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + test: + name: Test + runs-on: ubuntu-latest + needs: + - ensure-policyengine-bundle-supported-by-simulation-api + - test_env_vars + permissions: + contents: read + id-token: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Start Redis + run: sudo systemctl start redis-server + - name: Auth + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + with: + project_id: policyengine-api + - name: Install dependencies + run: make install + - name: Test the API + run: make test + env: + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: PolicyEngine/policyengine-api diff --git a/scripts/rules/tests/fixtures/policyengine_sim_api_pr.yml b/scripts/rules/tests/fixtures/policyengine_sim_api_pr.yml new file mode 100644 index 000000000..196fb5486 --- /dev/null +++ b/scripts/rules/tests/fixtures/policyengine_sim_api_pr.yml @@ -0,0 +1,179 @@ +name: PR checks + +on: + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + # Each entry is a uv project tested in its own locked environment. + # For the gateway this is the parity guarantee: the same lock the + # Modal image installs with uv_sync(frozen=True). + project: + - projects/policyengine-simulation-executor + - projects/policyengine-simulation-entry + - projects/policyengine-simulation-gateway + - libs/policyengine-simulation-contract + - libs/policyengine-simulation-observability + - libs/policyengine-stage12-persistence + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + run: | + cd ${{ matrix.project }} + uv sync --extra test + + - name: Run tests + run: | + cd ${{ matrix.project }} + uv run pytest tests/ -v + + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Install ruff + run: uv tool install ruff + + - name: Run ruff format check + run: | + for dir in projects/*/src libs/*/src; do + if [ -d "$dir" ]; then + echo "Checking format in $dir..." + uv run ruff format --check $dir + fi + done + + quality-guards: + name: Quality guards + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + # The rules registry guard compares baselines with the base branch. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Test the rules registry tool + run: python -m unittest discover -s scripts/rules/tests -t . + + - name: Run quality guards + run: python scripts/run_quality_guards.py + + docker-build: + name: Docker build + needs: [test, lint] # Only run if tests and linting pass + runs-on: ubuntu-latest + strategy: + matrix: + service: [simulation-entry, simulation-executor] + + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Build Docker image + uses: docker/build-push-action@v7 + with: + context: . + file: projects/policyengine-${{ matrix.service }}/Dockerfile + push: false + tags: policyengine-${{ matrix.service }}:test + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + test-integration: + name: Test integration + needs: docker-build # Only run if docker builds succeed + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Generate API clients + run: | + ./scripts/generate-clients.sh + + - name: Start services + run: | + docker compose -f deployment/docker-compose.yml up -d + # Wait for services to be ready + sleep 10 + # Check services are responding + for i in {1..30}; do + if curl -f http://localhost:8082/ping/alive; then + echo "Services are ready!" + break + fi + echo "Waiting for services... (attempt $i/30)" + sleep 2 + done + + - name: Run integration tests (local services only) + run: | + cd projects/policyengine-apis-integ + uv sync --extra test + # Run tests against local Docker services only (not Modal staging) + uv run pytest tests/ -v -m "not requires_gcp and not beta_only" + + - name: Show service logs on failure + if: failure() + run: | + docker compose -f deployment/docker-compose.yml logs + + - name: Stop services + if: always() + run: | + docker compose -f deployment/docker-compose.yml down diff --git a/scripts/rules/tests/support.py b/scripts/rules/tests/support.py new file mode 100644 index 000000000..d10653122 --- /dev/null +++ b/scripts/rules/tests/support.py @@ -0,0 +1,232 @@ +"""Build small throwaway repositories for exercising the rules tool.""" + +from __future__ import annotations + +import tempfile +import textwrap +from datetime import date +from pathlib import Path + +from scripts.rules import TOOL_VERSION +from scripts.rules.guard import GuardResult, run_guard +from scripts.rules.load import load_registry +from scripts.rules.render import write_outputs +from scripts.rules.resolve import clear_caches + +TODAY = date(2026, 6, 1) + +REGISTRY = f""" +schema_version = 1 +repo = "example" +tool_version = "{TOOL_VERSION}" + +[[category]] +id = "layers" +title = "Layers" + +[[category]] +id = "quality" +title = "Quality" + +[discovery] +guard_test_globs = ["tests/**/test_*boundar*.py"] + +[[rule]] +id = "ex.layers.no-sql-in-routes" +category = "layers" +statement = "Routes never issue SQL." +rationale = "docs/guide.md#routes" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "make test" + +[[rule]] +id = "ex.layers.services-no-http" +category = "layers" +statement = "Services never build HTTP responses." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::TestServices::test_no_http"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "make test" + +[[rule]] +id = "ex.process.registry" +category = "quality" +statement = "The registry is consistent." +rationale = "docs/guide.md" +scope = ["rules/registry.toml"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "quality-guard" +name = "rules-registry" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "quality-guards" +invocation = "run_quality_guards.py" + +[[rule]] +id = "ex.quality.format" +category = "quality" +statement = "Code is formatted." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = true +[rule.enforcer] +kind = "ruff" +mode = "format" +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "lint" +invocation = "ruff format --check" + +[[rule]] +id = "ex.quality.types" +category = "quality" +statement = "The package type-checks." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "mypy" +paths = ["pkg"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "typecheck" +invocation = "mypy" + +[[rule]] +id = "ex.layers.connectors-never-commit" +category = "layers" +statement = "Connectors never commit." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "doc-only" +autofix = false +[rule.enforcer] +kind = "none" +""" + +WORKFLOW = """ +name: Pull request +on: pull_request +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # make test is mentioned in this comment only in some cases + - name: Run tests + run: make test + lint: + runs-on: ubuntu-latest + steps: + - name: Format + run: | + pip install ruff + ruff format --check . + typecheck: + runs-on: ubuntu-latest + steps: + - run: uv run mypy + quality-guards: + runs-on: ubuntu-latest + steps: + - name: Guards + run: python scripts/run_quality_guards.py +""" + +FILES = { + "rules/registry.toml": REGISTRY, + ".github/workflows/pr.yml": WORKFLOW, + "Makefile": "test:\n\tpython -m pytest tests/unit --disable-pytest-warnings\n", + "pyproject.toml": '[tool.mypy]\nfiles = ["pkg"]\n', + "scripts/run_quality_guards.py": ('GUARDS = (("rules-registry", lambda: []),)\n'), + "tests/unit/test_boundaries.py": textwrap.dedent( + """ + import pytest + + + @pytest.mark.parametrize("path", ["a", "b"]) + def test_routes_have_no_sql(path): + pass + + + class TestServices: + def test_no_http(self): + pass + """ + ), + "docs/guide.md": "# Guide\n\n## Routes\n", + "pkg/routes.py": "def route():\n return 1\n", + "AGENTS.md": "# Agent Instructions\n\nRead the guide.\n", +} + + +class Repo: + """A temporary repository tree. Use as a context manager.""" + + def __init__(self, files: dict[str, str] | None = None, *, render: bool = True): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + for path, text in {**FILES, **(files or {})}.items(): + self.write(path, text) + if render: + self.render() + + def __enter__(self) -> Repo: + return self + + def __exit__(self, *exc: object) -> None: + self._tmp.cleanup() + + def write(self, path: str, text: str) -> None: + target = self.root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + clear_caches() + + def read(self, path: str) -> str: + return (self.root / path).read_text() + + def edit(self, path: str, old: str, new: str) -> None: + text = self.read(path) + assert old in text, f"{old!r} not in {path}" + self.write(path, text.replace(old, new, 1)) + + def render(self) -> None: + clear_caches() + registry, violations = load_registry(self.root) + assert registry is not None and not violations, violations + write_outputs(registry, self.root) + + def guard(self, **kwargs) -> GuardResult: + kwargs.setdefault("today", TODAY) + kwargs.setdefault("compare_base", "base_ref" in kwargs) + return run_guard(self.root, **kwargs) + + def messages(self, **kwargs) -> list[str]: + return [str(v) for v in self.guard(**kwargs).violations] diff --git a/scripts/rules/tests/test_guard.py b/scripts/rules/tests/test_guard.py new file mode 100644 index 000000000..4b98fa303 --- /dev/null +++ b/scripts/rules/tests/test_guard.py @@ -0,0 +1,613 @@ +"""Tests for the rules registry guard, using throwaway repositories.""" + +from __future__ import annotations + +import subprocess +import unittest +from datetime import date + +from scripts.rules.baseline import check_baselines +from scripts.rules.load import load_text +from scripts.rules.render import render_agents_block, render_rules_md +from scripts.rules.tests.support import REGISTRY, Repo + +RATCHETED_RULE = """ +[[rule]] +id = "ex.layers.ratcheted" +category = "layers" +statement = "Ratcheted rule." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "ratcheted" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "make test" +[rule.baseline] +file = "rules/baselines/ratcheted.txt" +count = {count} +""" + + +def with_rule(extra: str) -> dict[str, str]: + return {"rules/registry.toml": REGISTRY + extra} + + +class ValidRepositoryTest(unittest.TestCase): + def test_valid_repository_passes(self) -> None: + with Repo() as repo: + self.assertEqual(repo.messages(), []) + + def test_render_is_deterministic(self) -> None: + with Repo() as repo: + registry, _ = load_text(repo.read("rules/registry.toml")) + assert registry is not None + self.assertEqual(render_rules_md(registry), render_rules_md(registry)) + self.assertEqual( + render_agents_block(registry), render_agents_block(registry) + ) + + def test_rules_md_lists_gaps(self) -> None: + with Repo() as repo: + rules_md = repo.read("docs/generated/rules.md") + self.assertIn( + "ex.layers.connectors-never-commit", rules_md.split("## Gaps")[1] + ) + self.assertIn("```mermaid", rules_md) + self.assertIn("## Rules", repo.read("AGENTS.md")) + + +class SchemaTest(unittest.TestCase): + def assertViolation(self, repo: Repo, fragment: str) -> None: + messages = repo.messages() + self.assertTrue(any(fragment in m for m in messages), messages) + + def test_missing_required_field(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", 'statement = "Routes never issue SQL."\n', "" + ) + self.assertViolation(repo, "statement must be a non-empty string") + + def test_unknown_enforcer_kind(self) -> None: + with Repo(render=False) as repo: + repo.edit("rules/registry.toml", 'kind = "mypy"', 'kind = "flake8"') + self.assertViolation(repo, "enforcer.kind is 'flake8'") + + def test_bad_status(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", 'status = "doc-only"', 'status = "optional"' + ) + self.assertViolation(repo, "status is 'optional'") + + def test_unknown_key(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + "autofix = true", + 'autofix = true\nowner = "someone"', + ) + self.assertViolation(repo, "unknown key 'owner'") + + def test_duplicate_rule_id(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'id = "ex.quality.types"', + 'id = "ex.quality.format"', + ) + self.assertViolation(repo, "duplicate rule id") + + def test_tool_version_mismatch(self) -> None: + with Repo(render=False) as repo: + text = repo.read("rules/registry.toml") + repo.write( + "rules/registry.toml", + text.replace('tool_version = "', 'tool_version = "9', 1), + ) + self.assertViolation(repo, "tool_version is 9") + + def test_scope_glob_must_match(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'scope = ["rules/registry.toml"]', + 'scope = ["nowhere/*.py"]', + ) + self.assertViolation(repo, "scope 'nowhere/*.py' matches no files") + + def test_doc_only_rule_cannot_claim_enforcer(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'status = "enforced"\nautofix = true', + 'status = "doc-only"\nautofix = true', + ) + self.assertViolation(repo, "doc-only rules have no enforcer") + + def test_enforced_rule_needs_ci(self) -> None: + extra = """ +[[rule]] +id = "ex.layers.local-only" +category = "layers" +statement = "Only checked locally." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"] +""" + with Repo(with_rule(extra), render=False) as repo: + self.assertViolation(repo, "enforced rules must run in CI") + + def test_ruff_is_never_ratcheted(self) -> None: + with Repo(render=False) as repo: + repo.write("rules/baselines/ruff.txt", "pkg/routes.py E501\n") + repo.edit( + "rules/registry.toml", + 'kind = "ruff"\nmode = "format"', + 'kind = "ruff"\nmode = "format"\n[rule.baseline]\nfile = "rules/baselines/ruff.txt"\ncount = 1', + ) + self.assertViolation(repo, "ruff is never ratcheted") + + def test_ratcheted_without_baseline(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'status = "enforced"\nautofix = false\n[rule.enforcer]\nkind = "mypy"', + 'status = "ratcheted"\nautofix = false\n[rule.enforcer]\nkind = "mypy"', + ) + self.assertViolation(repo, "ratcheted rules need a baseline") + + def test_baseline_count_must_match_file(self) -> None: + with Repo(with_rule(RATCHETED_RULE.format(count=3)), render=False) as repo: + repo.write( + "rules/baselines/ratcheted.txt", + "# tolerated\npkg/routes.py\npkg/other.py\n", + ) + self.assertViolation( + repo, "baseline count is 3 but rules/baselines/ratcheted.txt lists 2" + ) + + def test_judgment_rules_need_review_enforcer(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'tier = "mechanical"\nstatus = "enforced"\nautofix = true', + 'tier = "judgment"\nstatus = "enforced"\nautofix = true', + ) + self.assertViolation(repo, "judgment rules are enforced by review") + + +class ResolutionTest(unittest.TestCase): + def assertViolation(self, repo: Repo, fragment: str) -> None: + messages = repo.messages() + self.assertTrue(any(fragment in m for m in messages), messages) + + def test_unresolved_pytest_node(self) -> None: + with Repo() as repo: + repo.edit( + "tests/unit/test_boundaries.py", + "def test_routes_have_no_sql", + "def test_renamed", + ) + self.assertViolation(repo, "has no test 'test_routes_have_no_sql'") + + def test_parametrized_node_resolves(self) -> None: + with Repo() as repo: + repo.edit( + "rules/registry.toml", + 'test_boundaries.py::test_routes_have_no_sql"', + 'test_boundaries.py::test_routes_have_no_sql[a]"', + ) + repo.render() + self.assertEqual(repo.messages(), []) + + def test_test_outside_make_target_is_not_collected(self) -> None: + with Repo() as repo: + repo.edit("Makefile", "tests/unit", "tests/contract") + self.assertViolation(repo, "is not collected by 'make test'") + + def test_project_must_be_in_matrix(self) -> None: + workflow = """ +jobs: + test: + strategy: + matrix: + project: + - projects/alpha + steps: + - run: | + cd ${{ matrix.project }} + uv run pytest tests/ +""" + extra = """ +[[rule]] +id = "ex.layers.beta" +category = "layers" +statement = "Beta rule." +rationale = "docs/guide.md" +scope = ["projects/beta/src/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "projects/beta" +nodes = ["tests/test_beta.py::test_beta"] +[[rule.ci]] +workflow = ".github/workflows/matrix.yml" +job = "test" +invocation = "pytest tests/" +""" + files = { + ".github/workflows/matrix.yml": workflow, + "projects/beta/tests/test_beta.py": "def test_beta():\n pass\n", + "projects/beta/src/beta.py": "", + **with_rule(extra), + } + with Repo(files) as repo: + self.assertViolation( + repo, "project projects/beta is not in the matrix of job 'test'" + ) + + def test_missing_job(self) -> None: + with Repo() as repo: + repo.edit("rules/registry.toml", 'job = "typecheck"', 'job = "types"') + repo.render() + self.assertViolation(repo, "no job 'types'") + + def test_invocation_only_in_comment_does_not_count(self) -> None: + with Repo() as repo: + repo.edit(".github/workflows/pr.yml", "run: make test", "run: echo skipped") + self.assertViolation(repo, "does not run 'make test'") + + def test_unknown_quality_guard(self) -> None: + with Repo() as repo: + repo.edit( + "rules/registry.toml", 'name = "rules-registry"', 'name = "renamed"' + ) + repo.render() + self.assertViolation(repo, "no guard named 'renamed'") + + def test_mypy_path_must_be_configured(self) -> None: + with Repo() as repo: + repo.edit( + "rules/registry.toml", 'paths = ["pkg"]', 'paths = ["pkg", "other"]' + ) + repo.render() + self.assertViolation(repo, "['other'] not in [tool.mypy].files") + + def test_ruff_lint_codes_must_be_selected(self) -> None: + extra = """ +[[rule]] +id = "ex.quality.bugbear" +category = "quality" +statement = "No bugbear findings." +rationale = "docs/guide.md" +scope = ["pkg/*.py"] +tier = "mechanical" +status = "advisory" +autofix = false +[rule.enforcer] +kind = "ruff" +mode = "lint" +codes = ["B"] +""" + with Repo(with_rule(extra)) as repo: + self.assertViolation(repo, "ruff does not select ['B']") + + def test_proof_files_must_exist(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'kind = "mypy"\npaths = ["pkg"]', + 'kind = "mypy"\npaths = ["pkg"]\n[rule.proof]\nbad_fixture = "rules/fixtures/bad.py"\n' + 'good_fixture = "rules/fixtures/good.py"\ntest = "tests/unit/test_boundaries.py::test_routes_have_no_sql"', + ) + self.assertViolation( + repo, "proof bad_fixture rules/fixtures/bad.py does not exist" + ) + + +class ProofCiTest(unittest.TestCase): + PROOF_FILES = { + "rules/fixtures/bad.py": "import sqlalchemy\n", + "rules/fixtures/good.py": "x = 1\n", + } + + def with_proof(self, repo: Repo, rule_marker: str, proof: str) -> None: + repo.edit( + "rules/registry.toml", rule_marker, rule_marker + "\n[rule.proof]\n" + proof + ) + repo.render() + + def test_pytest_rule_proof_runs_in_its_ci_job(self) -> None: + files = { + **self.PROOF_FILES, + "tests/unit/test_proof.py": "def test_rule_proof_routes():\n pass\n", + } + with Repo(files) as repo: + self.with_proof( + repo, + 'nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"]', + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tests/unit/test_proof.py::test_rule_proof_routes"', + ) + self.assertEqual(repo.messages(), []) + + def test_proof_outside_the_ci_command_is_rejected(self) -> None: + files = { + **self.PROOF_FILES, + "tests/contract/test_proof.py": "def test_rule_proof_routes():\n pass\n", + } + with Repo(files) as repo: + self.with_proof( + repo, + 'nodes = ["tests/unit/test_boundaries.py::test_routes_have_no_sql"]', + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tests/contract/test_proof.py::test_rule_proof_routes"', + ) + messages = "\n".join(repo.messages()) + self.assertIn( + "proof: tests/contract/test_proof.py is not collected by 'make test'", + messages, + ) + + def test_non_pytest_rule_proof_must_name_its_job(self) -> None: + files = { + **self.PROOF_FILES, + "tests/unit/test_proof.py": "def test_rule_proof_types():\n pass\n", + } + with Repo(files) as repo: + self.with_proof( + repo, + 'paths = ["pkg"]', + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tests/unit/test_proof.py::test_rule_proof_types"', + ) + self.assertIn( + "the proof test is not tied to a CI job", "\n".join(repo.messages()) + ) + + def test_proof_with_named_unittest_job(self) -> None: + workflow_job = ( + " tool-tests:\n steps:\n" + " - run: python -m unittest discover -s tools/tests -t .\n" + ) + files = { + **self.PROOF_FILES, + "tools/tests/test_proof.py": "class TestProof:\n def test_rule_proof_types(self):\n pass\n", + } + with Repo(files, render=False) as repo: + repo.write( + ".github/workflows/pr.yml", + repo.read(".github/workflows/pr.yml") + workflow_job, + ) + repo.edit( + "rules/registry.toml", + "[discovery]\n", + '[discovery]\nignore_jobs = [{ job = "tool-tests", reason = "runs tool tests" }]\n', + ) + proof = ( + 'bad_fixture = "rules/fixtures/bad.py"\ngood_fixture = "rules/fixtures/good.py"\n' + 'test = "tools/tests/test_proof.py::TestProof::test_rule_proof_types"\n' + 'workflow = ".github/workflows/pr.yml"\njob = "tool-tests"\n' + 'invocation = "unittest discover -s tools/tests"' + ) + self.with_proof(repo, 'paths = ["pkg"]', proof) + self.assertEqual(repo.messages(), []) + + repo.edit("rules/registry.toml", 'job = "tool-tests"\n', "") + repo.render() + self.assertIn( + "proof names its CI job only partly", "\n".join(repo.messages()) + ) + + +class HiddenRuleTest(unittest.TestCase): + def assertViolation(self, repo: Repo, fragment: str) -> None: + messages = repo.messages() + self.assertTrue(any(fragment in m for m in messages), messages) + + def test_unregistered_quality_guard(self) -> None: + with Repo() as repo: + repo.write( + "scripts/run_quality_guards.py", + 'GUARDS = (("rules-registry", lambda: []), ("secret-guard", lambda: []))\n', + ) + self.assertViolation(repo, "quality guard 'secret-guard'") + + def test_unregistered_guard_test(self) -> None: + with Repo() as repo: + text = repo.read("tests/unit/test_boundaries.py") + repo.write( + "tests/unit/test_boundaries.py", + text + "\n\ndef test_new_boundary():\n pass\n", + ) + self.assertViolation( + repo, "test_boundaries.py::test_new_boundary is not referenced" + ) + + def test_unregistered_mypy_file(self) -> None: + with Repo() as repo: + repo.write("pyproject.toml", '[tool.mypy]\nfiles = ["pkg", "scripts"]\n') + self.assertViolation(repo, "mypy checks 'scripts'") + + def test_unregistered_ruff_ignore(self) -> None: + with Repo() as repo: + repo.write( + "pyproject.toml", + '[tool.mypy]\nfiles = ["pkg"]\n\n[tool.ruff.lint.per-file-ignores]\n"pkg/*.py" = ["E501"]\n', + ) + self.assertViolation(repo, "ruff per-file-ignores 'pkg/*.py' has no rule") + + def test_unregistered_ci_job(self) -> None: + with Repo() as repo: + text = repo.read(".github/workflows/pr.yml") + repo.write( + ".github/workflows/pr.yml", + text + " deploy:\n steps:\n - run: ./deploy.sh\n", + ) + self.assertViolation(repo, "CI job 'deploy'") + + def test_ignored_ci_job_with_reason_passes(self) -> None: + with Repo() as repo: + text = repo.read(".github/workflows/pr.yml") + repo.write( + ".github/workflows/pr.yml", + text + " deploy:\n steps:\n - run: ./deploy.sh\n", + ) + repo.edit( + "rules/registry.toml", + "[discovery]\n", + '[discovery]\nignore_jobs = [{ job = "deploy", reason = "deploys, enforces nothing" }]\n', + ) + repo.render() + self.assertEqual(repo.messages(), []) + + +class RenderFreshnessTest(unittest.TestCase): + def test_stale_rules_md(self) -> None: + with Repo() as repo: + repo.write( + "docs/generated/rules.md", + repo.read("docs/generated/rules.md") + "edited\n", + ) + self.assertIn( + "[registry] render: docs/generated/rules.md is stale", + "\n".join(repo.messages()), + ) + + def test_missing_agents_markers(self) -> None: + with Repo() as repo: + repo.write("AGENTS.md", "# Agent Instructions\n") + self.assertIn( + "AGENTS.md has no generated rules block", "\n".join(repo.messages()) + ) + + def test_stale_agents_block(self) -> None: + with Repo() as repo: + repo.edit("AGENTS.md", "Routes never issue SQL.", "Routes may issue SQL.") + self.assertIn( + "the rules block in AGENTS.md is stale", "\n".join(repo.messages()) + ) + + def test_render_keeps_surrounding_agents_text(self) -> None: + with Repo() as repo: + agents = repo.read("AGENTS.md") + self.assertTrue( + agents.startswith("# Agent Instructions\n\nRead the guide.\n") + ) + repo.render() + self.assertEqual(repo.read("AGENTS.md"), agents) + + +class ExpiryTest(unittest.TestCase): + def test_expired_review_by(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'statement = "Code is formatted."', + 'statement = "Code is formatted."\nreview_by = 2026-01-01', + ) + repo.render() + messages = "\n".join(repo.messages()) + self.assertIn("review_by 2026-01-01 has passed", messages) + + def test_expired_exception(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'kind = "mypy"\npaths = ["pkg"]', + 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\npath = "pkg/routes.py"\n' + 'reason = "migrating"\nexpires = 2026-05-01', + ) + repo.render() + self.assertIn( + "exception for pkg/routes.py expired on 2026-05-01", + "\n".join(repo.messages()), + ) + self.assertEqual(repo.messages(today=date(2026, 4, 1)), []) + + def test_exception_is_listed_in_rules_md(self) -> None: + with Repo(render=False) as repo: + repo.edit( + "rules/registry.toml", + 'kind = "mypy"\npaths = ["pkg"]', + 'kind = "mypy"\npaths = ["pkg"]\n[[rule.exception]]\npath = "pkg/routes.py"\n' + 'reason = "migrating"\nexpires = 2026-12-01', + ) + repo.render() + self.assertIn( + "exception for `pkg/routes.py` until 2026-12-01", + repo.read("docs/generated/rules.md"), + ) + + +class BaselineTest(unittest.TestCase): + def registry(self, extra: str): + registry, violations = load_text(REGISTRY + extra) + assert registry is not None and not violations, violations + return registry + + def test_baseline_may_shrink(self) -> None: + base = self.registry(RATCHETED_RULE.format(count=3)) + current = self.registry(RATCHETED_RULE.format(count=2)) + self.assertEqual(check_baselines(current, base)[0], []) + + def test_baseline_may_not_grow(self) -> None: + base = self.registry(RATCHETED_RULE.format(count=2)) + current = self.registry(RATCHETED_RULE.format(count=3)) + violations = check_baselines(current, base)[0] + self.assertIn("baseline grew from 2 to 3", str(violations[0])) + + def test_downgrade_needs_reason(self) -> None: + base = self.registry("") + weakened = REGISTRY.replace( + 'status = "enforced"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\nmode = "format"\n[[rule.ci]]\n' + 'workflow = ".github/workflows/pr.yml"\njob = "lint"\ninvocation = "ruff format --check"', + 'status = "advisory"\nautofix = true\n[rule.enforcer]\nkind = "ruff"\nmode = "format"', + ) + current, _ = load_text(weakened) + assert current is not None + violations = check_baselines(current, base)[0] + self.assertIn("status weakened from enforced to advisory", str(violations[0])) + + with_reason, _ = load_text( + weakened.replace( + 'mode = "format"', + 'mode = "format"\n[rule.downgrade]\nreason = "flaky"', + 1, + ) + ) + assert with_reason is not None + self.assertEqual(check_baselines(with_reason, base)[0], []) + + def test_compares_against_git_base_ref(self) -> None: + with Repo(with_rule(RATCHETED_RULE.format(count=1)), render=False) as repo: + repo.write("rules/baselines/ratcheted.txt", "pkg/routes.py\n") + repo.render() + git = ["git", "-c", "user.email=t@example.com", "-c", "user.name=t"] + subprocess.run(["git", "init", "-q"], cwd=repo.root, check=True) + subprocess.run([*git, "add", "-A"], cwd=repo.root, check=True) + subprocess.run([*git, "commit", "-qm", "base"], cwd=repo.root, check=True) + repo.edit("rules/registry.toml", "count = 1", "count = 2") + repo.write("rules/baselines/ratcheted.txt", "pkg/routes.py\npkg/other.py\n") + repo.render() + self.assertIn( + "baseline grew from 1 to 2", "\n".join(repo.messages(base_ref="HEAD")) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/rules/tests/test_workflows.py b/scripts/rules/tests/test_workflows.py new file mode 100644 index 000000000..137337c52 --- /dev/null +++ b/scripts/rules/tests/test_workflows.py @@ -0,0 +1,80 @@ +"""Tests for the GitHub Actions subset reader, pinned against both repos' real PR workflows.""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +from scripts.rules.workflows import parse_jobs + +FIXTURES = Path(__file__).parent / "fixtures" + + +class RealWorkflowTest(unittest.TestCase): + def test_policyengine_api_pr_workflow(self) -> None: + jobs, problems = parse_jobs((FIXTURES / "policyengine_api_pr.yml").read_text()) + self.assertEqual(problems, []) + self.assertIn("make test", jobs["test"].run_text) + self.assertIn("ruff format --check", jobs["lint"].run_text) + self.assertIn("run_quality_guards.py", jobs["quality-guards"].run_text) + self.assertIn("unittest discover", jobs["quality-guards"].run_text) + self.assertTrue(jobs["alembic-v1-check"].invokes("alembic-v1-check.yml")) + self.assertEqual(jobs["typecheck-v2"].name, "Type-check API v2") + + def test_policyengine_sim_api_pr_workflow(self) -> None: + jobs, problems = parse_jobs( + (FIXTURES / "policyengine_sim_api_pr.yml").read_text() + ) + self.assertEqual(problems, []) + self.assertIn( + "projects/policyengine-simulation-entry", jobs["test"].matrix["project"] + ) + self.assertIn( + "libs/policyengine-stage12-persistence", jobs["test"].matrix["project"] + ) + self.assertIn("uv run pytest tests/", jobs["test"].run_text) + self.assertIn("ruff format --check", jobs["lint"].run_text) + self.assertIn("run_quality_guards.py", jobs["quality-guards"].run_text) + self.assertTrue(jobs["docker-build"].invokes("docker/build-push-action")) + + +class SubsetTest(unittest.TestCase): + def test_block_scalar_stops_at_sibling_key(self) -> None: + text = """ +jobs: + build: + steps: + - name: Build + run: | + make build + env: + FLAG: make test +""" + jobs, _ = parse_jobs(text) + self.assertFalse(jobs["build"].invokes("make test")) + self.assertTrue(jobs["build"].invokes("make build")) + + def test_comments_are_ignored(self) -> None: + text = "jobs:\n a:\n steps:\n # run: make test\n - run: echo hi # make test\n" + jobs, _ = parse_jobs(text) + self.assertFalse(jobs["a"].invokes("make test")) + + def test_quoted_hash_is_not_a_comment(self) -> None: + jobs, _ = parse_jobs( + 'jobs:\n a:\n steps:\n - run: "echo #1 && make test"\n' + ) + self.assertTrue(jobs["a"].invokes("make test")) + + def test_inline_matrix_list(self) -> None: + jobs, _ = parse_jobs( + 'jobs:\n a:\n strategy:\n matrix:\n python: ["3.12", "3.13"]\n' + ) + self.assertEqual(jobs["a"].matrix["python"], ("3.12", "3.13")) + + def test_anchors_are_reported(self) -> None: + _, problems = parse_jobs("jobs:\n a: &base\n steps: []\n") + self.assertTrue(problems) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/rules/validate.py b/scripts/rules/validate.py new file mode 100644 index 000000000..011a33c66 --- /dev/null +++ b/scripts/rules/validate.py @@ -0,0 +1,333 @@ +"""Semantic validation of a parsed registry against the repository tree.""" + +from __future__ import annotations + +import re +from datetime import date +from pathlib import Path +from typing import Any + +from scripts.rules import TOOL_VERSION +from scripts.rules.model import Kind, Registry, Rule, Status, Tier, Violation +from scripts.rules.resolve import node_exists + +RULE_ID = re.compile(r"^[a-z0-9]+(\.[a-z0-9][a-z0-9-]*)+$") + +# Required and optional enforcer fields per kind, with their expected types. +KIND_FIELDS: dict[Kind, tuple[dict[str, type], dict[str, type]]] = { + Kind.PYTEST: ({"nodes": list}, {"project": str}), + Kind.QUALITY_GUARD: ({"name": str}, {"runner": str}), + Kind.RUFF: ({"mode": str}, {"codes": list, "pattern": str, "config": str}), + Kind.MYPY: ({"paths": list}, {"config": str}), + Kind.PYRIGHT: ({"config": str}, {"project": str}), + Kind.IMPORT_LINTER: ({"config": str, "contract": str}, {}), + Kind.WORKFLOW_JOB: ({}, {"step": str}), + Kind.DOCS_GUARD: ({"doc": str, "marker": str}, {}), + Kind.LLM_REVIEW: ({"prompt": str, "model": str}, {}), + Kind.NONE: ({}, {}), +} + +RUFF_MODES = ("format", "lint", "per-file-ignore") + +PROOF_FIELDS: dict[Tier, dict[str, Any]] = { + Tier.MECHANICAL: {"bad_fixture": str, "good_fixture": str, "test": str}, + Tier.METRIC: { + "below_fixture": str, + "above_fixture": str, + "threshold": (int, float), + "test": str, + }, + Tier.JUDGMENT: { + "eval_set": str, + "min_precision": float, + "measured_precision": float, + "measured_on": date, + }, +} +PROOF_FILE_FIELDS = ( + "bad_fixture", + "good_fixture", + "below_fixture", + "above_fixture", + "eval_set", +) + + +def baseline_entries(path: Path) -> int: + return sum( + 1 + for line in path.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + +def _check_fields( + rule: Rule, + fields: dict[str, Any], + required: dict[str, Any], + optional: dict[str, Any], + where: str, +) -> list[Violation]: + violations = [] + for key, expected in required.items(): + if key not in fields: + violations.append( + Violation( + rule.id, "schema", f"{where} is missing '{key}'", f"add {key}" + ) + ) + elif not isinstance(fields[key], expected): + violations.append( + Violation( + rule.id, + "schema", + f"{where}.{key} has the wrong type", + f"make it a {expected}", + ) + ) + for key in sorted(set(fields) - set(required) - set(optional)): + violations.append( + Violation( + rule.id, + "schema", + f"unknown key '{key}' in {where}", + f"remove it; allowed: {sorted({**required, **optional})}", + ) + ) + return violations + + +def _validate_enforcer(rule: Rule) -> list[Violation]: + required, optional = KIND_FIELDS[rule.enforcer.kind] + where = f"enforcer ({rule.enforcer.kind})" + violations = _check_fields( + rule, dict(rule.enforcer.fields), required, optional, where + ) + kind, fields = rule.enforcer.kind, rule.enforcer.fields + if kind is Kind.PYTEST and not fields.get("nodes"): + violations.append( + Violation( + rule.id, + "schema", + "pytest enforcer has no nodes", + "list at least one test node", + ) + ) + if kind is Kind.RUFF: + mode = fields.get("mode") + if mode not in RUFF_MODES: + violations.append( + Violation( + rule.id, + "schema", + f"ruff mode is {mode!r}", + f"use one of {list(RUFF_MODES)}", + ) + ) + if mode in ("lint", "per-file-ignore") and not fields.get("codes"): + violations.append( + Violation( + rule.id, "schema", f"ruff {mode} needs codes", "list the ruff codes" + ) + ) + if mode == "per-file-ignore" and not fields.get("pattern"): + violations.append( + Violation( + rule.id, + "schema", + "ruff per-file-ignore needs a pattern", + "set pattern", + ) + ) + return violations + + +def _validate_cross_fields(rule: Rule) -> list[Violation]: + violations = [] + + def v(message: str, fix: str) -> None: + violations.append(Violation(rule.id, "schema", message, fix)) + + kind, status, tier = rule.enforcer.kind, rule.status, rule.tier + if status is Status.DOC_ONLY and kind is not Kind.NONE: + v( + "doc-only rules have no enforcer", + 'set enforcer.kind = "none", or pick a stronger status', + ) + if status is Status.DOC_ONLY and rule.ci: + v( + "doc-only rules have no CI wiring", + "remove rule.ci, or pick a stronger status", + ) + if kind is Kind.NONE and status not in (Status.DOC_ONLY, Status.ADVISORY): + v( + f"status {status} needs an enforcer", + "add [rule.enforcer] or use doc-only/advisory", + ) + if status in (Status.ENFORCED, Status.RATCHETED) and not rule.ci: + v(f"{status} rules must run in CI", "add a [[rule.ci]] reference") + if status is Status.RATCHETED and rule.baseline is None: + v("ratcheted rules need a baseline", "add [rule.baseline] or use enforced") + if rule.baseline is not None and status is not Status.RATCHETED: + v( + "only ratcheted rules may have a baseline", + 'remove [rule.baseline] or set status = "ratcheted"', + ) + if kind is Kind.RUFF and rule.baseline is not None: + v( + "ruff is never ratcheted", + "remove the baseline and fix every violation in a standalone PR that also changes the ruff select list", + ) + if tier is Tier.JUDGMENT and kind not in (Kind.LLM_REVIEW, Kind.NONE): + v( + "judgment rules are enforced by review, not by a mechanical checker", + 'use enforcer.kind = "llm-review" or "none"', + ) + if kind is Kind.LLM_REVIEW and tier is not Tier.JUDGMENT: + v("llm-review only enforces judgment rules", 'set tier = "judgment"') + return violations + + +def _validate_proof(rule: Rule, root: Path) -> list[Violation]: + if rule.proof is None: + return [] + fields = dict(rule.proof.fields) + optional = ( + {"project": str, "workflow": str, "job": str, "invocation": str} + if "test" in PROOF_FIELDS[rule.tier] + else {} + ) + violations = _check_fields( + rule, fields, PROOF_FIELDS[rule.tier], optional, f"proof ({rule.tier})" + ) + named = [key for key in ("workflow", "job", "invocation") if key in fields] + if named and len(named) != 3: + violations.append( + Violation( + rule.id, + "schema", + "proof names its CI job only partly", + "set all of proof.workflow, proof.job and proof.invocation, or none", + ) + ) + for key in PROOF_FILE_FIELDS: + if isinstance(fields.get(key), str) and not (root / fields[key]).exists(): + violations.append( + Violation( + rule.id, + "proof", + f"proof {key} {fields[key]} does not exist", + "add the fixture or fix the path", + ) + ) + test = fields.get("test") + if isinstance(test, str): + prefix = ( + f"{fields['project'].rstrip('/')}/" + if isinstance(fields.get("project"), str) + else "" + ) + problem = node_exists(root, prefix + test) + if problem: + violations.append( + Violation( + rule.id, + "proof", + problem, + "point proof.test at the test that runs the fixtures", + ) + ) + return violations + + +def validate(registry: Registry, root: Path, today: date) -> list[Violation]: + violations: list[Violation] = [] + + def v(rule_id: str | None, message: str, fix: str, check: str = "schema") -> None: + violations.append(Violation(rule_id, check, message, fix)) + + if registry.tool_version != TOOL_VERSION: + v( + None, + f"tool_version is {registry.tool_version} but scripts/rules is {TOOL_VERSION}", + "sync scripts/rules (see SYNC.md) and update tool_version", + ) + + category_ids = [category.id for category in registry.categories] + for duplicate in sorted({c for c in category_ids if category_ids.count(c) > 1}): + v(None, f"duplicate category '{duplicate}'", "give each category a unique id") + + rule_ids = [rule.id for rule in registry.rules] + for duplicate in sorted({r for r in rule_ids if rule_ids.count(r) > 1}): + v(duplicate, "duplicate rule id", "give each rule a unique id") + + for rule in registry.rules: + if rule.id and not RULE_ID.match(rule.id): + v( + rule.id, + "id must be lowercase dotted, e.g. 'api.v2.routes-no-sql'", + "rename the rule", + ) + if rule.category not in category_ids: + v( + rule.id, + f"unknown category '{rule.category}'", + f"use one of {category_ids} or add a [[category]]", + ) + + rationale_path = rule.rationale.split("#", 1)[0] + if rationale_path and not (root / rationale_path).exists(): + v( + rule.id, + f"rationale {rationale_path} does not exist", + "link an existing doc", + ) + + for pattern in rule.scope: + if not any(root.glob(pattern)): + v(rule.id, f"scope '{pattern}' matches no files", "fix the glob") + + violations.extend(_validate_enforcer(rule)) + violations.extend(_validate_cross_fields(rule)) + violations.extend(_validate_proof(rule, root)) + + if rule.baseline is not None: + path = root / rule.baseline.file + if not path.is_file(): + v( + rule.id, + f"baseline file {rule.baseline.file} does not exist", + "create it", + "baseline", + ) + elif baseline_entries(path) != rule.baseline.count: + v( + rule.id, + f"baseline count is {rule.baseline.count} but {rule.baseline.file} lists {baseline_entries(path)}", + "make the count match the file", + "baseline", + ) + + if rule.review_by is not None and rule.review_by < today: + v( + rule.id, + f"review_by {rule.review_by} has passed", + "review the rule, then remove or extend review_by", + "expiry", + ) + for exception in rule.exceptions: + if exception.expires < today: + v( + rule.id, + f"exception for {exception.path} expired on {exception.expires}", + "remove the exception (fix the code) or extend it with a reason", + "expiry", + ) + if not any(root.glob(exception.path)): + v( + rule.id, + f"exception path '{exception.path}' matches no files", + "remove the stale exception", + ) + + return violations diff --git a/scripts/rules/workflows.py b/scripts/rules/workflows.py new file mode 100644 index 000000000..2c3555f9d --- /dev/null +++ b/scripts/rules/workflows.py @@ -0,0 +1,191 @@ +"""Minimal reader for GitHub Actions workflow files. + +The standard library has no YAML parser and the guard must run without +dependencies, so this reads only the subset these repositories use: block-style +``jobs:`` mappings, ``name:``/``uses:``/``run:`` keys (including ``|`` and ``>`` +block scalars), and ``strategy.matrix`` lists. Anything outside that subset is +reported as unsupported rather than guessed at. + +It answers two questions: does job X exist, and does its run text (or reusable +workflow reference) contain a given invocation string? +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +_KEY = re.compile(r"^(\s*)(-\s+)?([A-Za-z0-9_-]+):\s*(.*)$") +_LIST_ITEM = re.compile(r"^(\s*)-\s+(.*)$") +_UNSUPPORTED = re.compile(r"(<<:|:\s*[&*][A-Za-z0-9_-]+\s*$)") + + +@dataclass(frozen=True) +class JobBlock: + id: str + name: str + run_text: str + uses: str + step_names: tuple[str, ...] + step_uses: tuple[str, ...] = () + matrix: dict[str, tuple[str, ...]] = field(default_factory=dict) + + def invokes(self, invocation: str) -> bool: + """Whether the job's commands or actions mention ``invocation``.""" + return ( + invocation in self.run_text + or invocation in self.uses + or any(invocation in action for action in self.step_uses) + ) + + +def _strip_comment(line: str) -> str: + in_single = in_double = False + for index, char in enumerate(line): + if char == "'" and not in_double: + in_single = not in_single + elif char == '"' and not in_single: + in_double = not in_double + elif char == "#" and not in_single and not in_double: + if index == 0 or line[index - 1].isspace(): + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"": + return value[1:-1] + return value + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def _inline_list(value: str) -> tuple[str, ...]: + inner = value.strip()[1:-1] + return tuple(_unquote(item) for item in inner.split(",") if item.strip()) + + +def _parse_job(job_id: str, lines: list[str], job_indent: int) -> JobBlock: + name = "" + uses = "" + step_uses: list[str] = [] + runs: list[str] = [] + steps: list[str] = [] + matrix: dict[str, tuple[str, ...]] = {} + body_indent = None + index = 0 + while index < len(lines): + line = lines[index] + match = _KEY.match(line) + if not match: + index += 1 + continue + leading, dash, key, value = match.groups() + key_column = len(leading) + (len(dash) if dash else 0) + if body_indent is None and not dash: + body_indent = key_column + top_level = not dash and key_column == body_indent + + if key == "name" and top_level: + name = _unquote(value) + elif key == "name": + steps.append(_unquote(value)) + elif key == "uses" and top_level: + uses = _unquote(value) + elif key == "uses": + step_uses.append(_unquote(value)) + elif key == "run": + if value.strip()[:1] in ("|", ">"): + block = [] + index += 1 + while index < len(lines) and ( + not lines[index].strip() or _indent(lines[index]) > key_column + ): + block.append(lines[index].strip()) + index += 1 + runs.append("\n".join(block)) + continue + runs.append(_unquote(value)) + elif key == "matrix" and not value: + index += 1 + while index < len(lines) and ( + not lines[index].strip() or _indent(lines[index]) > key_column + ): + entry = _KEY.match(lines[index]) + if entry and not entry.group(2): + entry_key, entry_value = entry.group(3), entry.group(4) + entry_column = len(entry.group(1)) + if entry_value.startswith("["): + matrix[entry_key] = _inline_list(entry_value) + index += 1 + continue + items = [] + index += 1 + while index < len(lines) and ( + not lines[index].strip() + or _indent(lines[index]) >= entry_column + ): + item = _LIST_ITEM.match(lines[index]) + if not item or _KEY.match(lines[index]): + break + items.append(_unquote(item.group(2))) + index += 1 + matrix[entry_key] = tuple(items) + continue + index += 1 + continue + index += 1 + return JobBlock( + id=job_id, + name=name or job_id, + run_text="\n".join(runs), + uses=uses, + step_uses=tuple(step_uses), + step_names=tuple(steps), + matrix=matrix, + ) + + +def parse_jobs(text: str) -> tuple[dict[str, JobBlock], list[str]]: + """Return the jobs in a workflow, plus descriptions of unsupported constructs.""" + lines = [_strip_comment(line) for line in text.splitlines()] + problems = [ + f"line {number}: YAML anchors/aliases/merge keys are not supported" + for number, line in enumerate(lines, 1) + if _UNSUPPORTED.search(line) + ] + + start = next( + (i for i, line in enumerate(lines) if re.match(r"^jobs:\s*$", line)), None + ) + if start is None: + if any(re.match(r"^jobs:\s*\S", line) for line in lines): + problems.append("flow-style 'jobs:' mapping is not supported") + return {}, problems + + body: list[str] = [] + for line in lines[start + 1 :]: + if line.strip() and _indent(line) == 0: + break + body.append(line) + + job_indent = next((_indent(line) for line in body if line.strip()), 0) + jobs: dict[str, JobBlock] = {} + current: str | None = None + current_lines: list[str] = [] + for line in body: + match = re.match(rf"^ {{{job_indent}}}([A-Za-z0-9_-]+):\s*$", line) + if match and _indent(line) == job_indent: + if current: + jobs[current] = _parse_job(current, current_lines, job_indent) + current, current_lines = match.group(1), [] + elif line.strip() and _indent(line) == job_indent: + problems.append(f"unexpected line at job level: {line.strip()!r}") + elif current: + current_lines.append(line) + if current: + jobs[current] = _parse_job(current, current_lines, job_indent) + return jobs, problems diff --git a/scripts/run_quality_guards.py b/scripts/run_quality_guards.py new file mode 100644 index 000000000..0338c10d1 --- /dev/null +++ b/scripts/run_quality_guards.py @@ -0,0 +1,39 @@ +"""Run repository quality guards.""" + +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from scripts.rules import guard as rules_guard # noqa: E402 + + +Guard = tuple[str, Callable[[], list[str]]] + +GUARDS: tuple[Guard, ...] = (("rules-registry", rules_guard.check),) + + +def main() -> int: + failed = False + for name, check in GUARDS: + violations = check() + if not violations: + print(f"{name}: passed") + continue + + failed = True + print(f"{name}: failed") + for violation in violations: + print(f" - {violation}") + + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main())