Enable ruff structure and import rules (INP, PLC, SLF, N, A, FAST, E)

.github/scripts/update_policyengine_package.py

@@ -201,7 +201,9 @@ def latest_version(package: str) -> None:
 
 def bundle_versions() -> None:
     """Print the package and data versions selected by PolicyEngine.py."""
-    from policyengine.bundle import get_current_bundle
+    from policyengine.bundle import (  # noqa: PLC0415  # only bundle-versions runs under uv with policyengine installed; the other commands use the runner's python3
+        get_current_bundle,
+    )
 
     bundle = _mapping(get_current_bundle())
     packages = _mapping(bundle.get("packages"))
@@ -247,7 +249,7 @@ def dataset_sha256(country: str) -> str:
     def certificate_sha256(country: str) -> str:
         # Receipt-v1 certificate digest; older wrappers lack the helper.
         try:
-            from policyengine.provenance.receipt import (
+            from policyengine.provenance.receipt import (  # noqa: PLC0415  # older policyengine wrappers lack the receipt helper, so the ImportError is handled here
                 certificate_sha256 as digest,
             )
         except ImportError:

AGENTS.md

@@ -135,13 +135,13 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 
 ### Code quality and tests
 
-- `sim.quality.deployed-tests-ruff-exemptions`: The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (S101, D1, ANN, PLR2004); every other ruff rule applies to it. (blocking: ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/authenticated_tests/**`)
+- `sim.quality.deployed-tests-ruff-exemptions`: The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to it. (blocking: ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/authenticated_tests/**`)
 - `sim.quality.every-project-tested`: Every project and library with unit tests runs them in the PR test matrix. (doc-only, not checked: follow it yourself)
 - `sim.quality.ruff-format-src`: Source code under projects/*/src and libs/*/src is formatted with ruff format. (blocking: ruff format)
 - `sim.quality.ruff-format-tests`: Test code is formatted with ruff format too; CI currently checks only src directories. (doc-only, not checked: follow it yourself)
-- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W))
+- `sim.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W))
 - `sim.quality.tests-pass`: Each project's unit tests pass in its own locked environment. (blocking: CI job `test`)
-- `sim.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**`)
+- `sim.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/tests/**`)
 
 ### CI workflows and scripts
 

changelog_entry.yaml

@@ -4,6 +4,7 @@
       - Made CLAUDE.md and the Copilot instructions thin pointers to AGENTS.md, required workflow logic longer than one line to live in tested and shellchecked scripts, and registered the tracking-issue rule
       - Added a rules registry (rules/registry.toml) with a standard-library tool that validates it, checks every rule is enforced where it claims, and renders docs/generated/rules.md and the Rules block in AGENTS.md
     changed:
+      - Enabled ruff's structure and import rules (INP, PLC, SLF, N, A, FAST, E); long lines are wrapped, deferred imports that Modal images, test seams, import cycles or heavy dependencies need carry a reason, script directories are declared to ruff as namespace packages, four internal exceptions gained an Error suffix, and published field names, OpenAPI documents and logged error types are unchanged; tests are also exempt from SLF001, INP001 and PLC0415
       - Enabled ruff's security, logging and debugging rules (S, T20, LOG, G, DTZ, ERA, TD, FIX, EXE); asserts outside tests became explicit checks, command-line scripts keep their printed output, test subprocess calls go through helpers, and every deliberate exception carries a reasoned suppression; the gateway's auth misconfiguration log no longer repeats its message, and the deployed-service tests share the unit-test ruff exemptions
       - Enabled ruff's exception rules (TRY, EM, BLE); exception messages are built before the raise, raises inside try blocks moved into helpers, and every deliberate catch-all or traceback-free error log carries a reasoned suppression; message text is unchanged, and error types change only in an operator script's two type checks
       - Enabled ruff's docstring rules (D) and documented every module, class and function outside tests; published contract models and routes now carry these descriptions in their JSON schema and OpenAPI, with no structural schema change

docs/generated/rules.md

@@ -242,43 +242,43 @@ flowchart LR
 
 | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |
 | --- | --- | --- | --- | --- | --- | --- | --- |
-| `sim.quality.deployed-tests-ruff-exemptions` | The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (S101, D1, ANN, PLR2004); every other ruff rule applies to it. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/authenticated_tests/**` | `pr.yml:lint` | — |
+| `sim.quality.deployed-tests-ruff-exemptions` | The deployed-service test suite (projects/*/authenticated_tests, pytest run against a live candidate) gets the same ruff exemptions as unit tests (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to it. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/authenticated_tests/**` | `pr.yml:lint` | — |
 | `sim.quality.every-project-tested` | Every project and library with unit tests runs them in the PR test matrix. | mechanical | doc-only | no | nothing | — | — |
 | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — |
 | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — |
-| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — |
+| `sim.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in ruff.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — |
 | `sim.quality.tests-pass` | Each project's unit tests pass in its own locked environment. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — |
-| `sim.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**` | `pr.yml:lint` | — |
+| `sim.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/tests/**` | `pr.yml:lint` | — |
 
 ```mermaid
 flowchart LR
   r_sim_quality_deployed_tests_ruff_exemptions("sim.quality.deployed-tests-ruff-exemptions (enforced)")
-  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____authenticated_tests___["ruff per-file ignore of ANN, D1, PLR2004, S101 for **/authenticated_tests/**"]
+  e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____authenticated_tests___["ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for **/authenticated_tests/**"]
   j_CI_pr_yml_lint["CI pr.yml:lint"]
   r_sim_quality_every_project_tested("sim.quality.every-project-tested (doc-only)")
   gap_unenforced>"not enforced"]
   r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)")
   e_ruff_format["ruff format"]
   r_sim_quality_ruff_format_tests("sim.quality.ruff-format-tests (doc-only)")
   r_sim_quality_ruff_lint("sim.quality.ruff-lint (enforced)")
-  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_["ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W)"]
+  e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_["ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W)"]
   r_sim_quality_tests_pass("sim.quality.tests-pass (enforced)")
   e_CI_job_test["CI job test"]
   j_CI_pr_yml_test["CI pr.yml:test"]
   r_sim_quality_tests_ruff_exemptions("sim.quality.tests-ruff-exemptions (enforced)")
-  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___["ruff per-file ignore of ANN, D1, PLR2004, S101 for **/tests/**"]
-  r_sim_quality_deployed_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____authenticated_tests___
-  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____authenticated_tests___ --> j_CI_pr_yml_lint
+  e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___["ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for **/tests/**"]
+  r_sim_quality_deployed_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____authenticated_tests___
+  e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____authenticated_tests___ --> j_CI_pr_yml_lint
   r_sim_quality_every_project_tested -.-> gap_unenforced
   r_sim_quality_ruff_format_src --> e_ruff_format
   e_ruff_format --> j_CI_pr_yml_lint
   r_sim_quality_ruff_format_tests -.-> gap_unenforced
-  r_sim_quality_ruff_lint --> e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_
-  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint
+  r_sim_quality_ruff_lint --> e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_
+  e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint
   r_sim_quality_tests_pass --> e_CI_job_test
   e_CI_job_test --> j_CI_pr_yml_test
-  r_sim_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___
-  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___ --> j_CI_pr_yml_lint
+  r_sim_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___
+  e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___ --> j_CI_pr_yml_lint
 ```
 
 ## CI workflows and scripts

libs/policyengine-fastapi/src/policyengine_fastapi/exit.py

@@ -47,4 +47,4 @@ async def lifespan(self) -> AsyncIterator[None]:
         self._exit()
 
 
-exit = AppExit()
+exit = AppExit()  # noqa: A001  # public name: projects import policyengine_fastapi.exit.exit

libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py

@@ -1,4 +1,4 @@
-"""Shared observability contracts and utilities for the PolicyEngine simulation API services.
+"""Observability contracts and utilities shared by the PolicyEngine simulation APIs.
 
 This stays in `policyengine-fastapi` for now because only one active service
 needs it and the main value today is shared contract stability, not a separate

libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/console.py

@@ -15,7 +15,7 @@
     ConsoleSpanExporter,
 )
 
-from policyengine_fastapi.exit import exit
+from policyengine_fastapi.exit import exit as app_exit
 
 
 # Configure opentelemetry
@@ -27,16 +27,16 @@ def export_ot_to_console(resource: Resource) -> None:
     Installs global tracer and meter providers for ``resource`` and registers an
     ``exit`` callback that flushes both at shutdown.
     """
-    traceProvider = TracerProvider(resource=resource)
+    trace_provider = TracerProvider(resource=resource)
     processor = BatchSpanProcessor(ConsoleSpanExporter())
-    traceProvider.add_span_processor(processor)
-    trace.set_tracer_provider(traceProvider)
+    trace_provider.add_span_processor(processor)
+    trace.set_tracer_provider(trace_provider)
 
     reader = PeriodicExportingMetricReader(ConsoleMetricExporter())
-    meterProvider = MeterProvider(resource=resource, metric_readers=[reader])
-    metrics.set_meter_provider(meterProvider)
+    meter_provider = MeterProvider(resource=resource, metric_readers=[reader])
+    metrics.set_meter_provider(meter_provider)
 
-    @exit()
+    @app_exit()
     def flush_ot() -> None:
-        traceProvider.force_flush()
-        meterProvider.force_flush()
+        trace_provider.force_flush()
+        meter_provider.force_flush()

libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py

@@ -10,7 +10,7 @@
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
-from policyengine_fastapi.exit import exit
+from policyengine_fastapi.exit import exit as app_exit
 
 log = logging.getLogger(__name__)
 
@@ -49,7 +49,9 @@ def _get_project_id() -> str:
 
 
 def _get_project_id_from_metadata() -> str | None:
-    from urllib import request
+    from urllib import (  # noqa: PLC0415  # urllib.request loads only when the metadata probe runs
+        request,
+    )
 
     try:
         return (
@@ -91,12 +93,17 @@ def instrument(self) -> None:
         Uses ``logging.basicConfig``, so the handler is not installed if the root
         logger already has handlers.
         """
-        from opentelemetry.instrumentation.logging import LoggingInstrumentor
-        from pythonjsonlogger.json import JsonFormatter
+        from opentelemetry.instrumentation.logging import (  # noqa: PLC0415  # the logging instrumentation loads only when GCP logging is set up
+            LoggingInstrumentor,
+        )
+        from pythonjsonlogger.json import (  # noqa: PLC0415  # the JSON formatter loads only when GCP logging is set up
+            JsonFormatter,
+        )
 
-        logHandler = logging.StreamHandler()
+        log_handler = logging.StreamHandler()
         formatter = JsonFormatter(
-            "%(asctime)s %(levelname)s %(message)s %(otelTraceID)s %(otelSpanID)s %(otelTraceSampled)s",
+            "%(asctime)s %(levelname)s %(message)s "
+            "%(otelTraceID)s %(otelSpanID)s %(otelTraceSampled)s",
             rename_fields={
                 "levelname": "severity",
                 "asctime": "timestamp",
@@ -107,12 +114,12 @@ def instrument(self) -> None:
             datefmt="%Y-%m-%dT%H:%M:%SZ",
         )
 
-        logHandler.setFormatter(formatter)
-        logHandler.addFilter(AddGcpProjectToTraceFilter(self.project_id))
+        log_handler.setFormatter(formatter)
+        log_handler.addFilter(AddGcpProjectToTraceFilter(self.project_id))
 
         logging.basicConfig(
             level=logging.INFO,
-            handlers=[logHandler],
+            handlers=[log_handler],
         )
         LoggingInstrumentor().instrument()
 
@@ -124,23 +131,25 @@ def export_ot_to_gcp(resource: Resource) -> None:
     providers for ``resource`` and registers an ``exit`` callback that flushes
     both at shutdown.
     """
-    from opentelemetry.exporter.cloud_monitoring import (
+    from opentelemetry.exporter.cloud_monitoring import (  # noqa: PLC0415  # the Google Cloud exporters load only when GCP export is set up
         CloudMonitoringMetricsExporter,
     )
-    from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter
+    from opentelemetry.exporter.cloud_trace import (  # noqa: PLC0415  # the Google Cloud exporters load only when GCP export is set up
+        CloudTraceSpanExporter,
+    )
 
-    traceProvider = TracerProvider(resource=resource)
+    trace_provider = TracerProvider(resource=resource)
     processor = BatchSpanProcessor(CloudTraceSpanExporter())
-    traceProvider.add_span_processor(processor)
-    trace.set_tracer_provider(traceProvider)
+    trace_provider.add_span_processor(processor)
+    trace.set_tracer_provider(trace_provider)
 
     reader = PeriodicExportingMetricReader(CloudMonitoringMetricsExporter())
-    meterProvider = MeterProvider(metric_readers=[reader], resource=resource)
-    metrics.set_meter_provider(meterProvider)
+    meter_provider = MeterProvider(metric_readers=[reader], resource=resource)
+    metrics.set_meter_provider(meter_provider)
 
-    @exit()
+    @app_exit()
     def flush_ot() -> None:
         log.info("Flushing traces and metrics before shutdown")
-        traceProvider.force_flush()
-        meterProvider.force_flush()
+        trace_provider.force_flush()
+        meter_provider.force_flush()
         log.info("All metics and traces flushed.")

libs/policyengine-fastapi/src/policyengine_fastapi/ping/__init__.py

@@ -28,7 +28,7 @@ async def ping(request: PingRequest) -> PingResponse:
         """Verify the API can receive and process unauthenticated requests."""
         return PingResponse(incremented=request.value + 1)
 
-    @router.get("/ping/started", response_model=str)
+    @router.get("/ping/started")
     async def started() -> str:
         """Verify the api is running."""
         return "alive"

libs/policyengine-fastapi/tests/test_exit.py

@@ -1,7 +1,7 @@
 from unittest.mock import Mock, call
 
 import pytest
-from policyengine_fastapi.exit import exit
+from policyengine_fastapi.exit import exit as app_exit
 
 
 @pytest.mark.asyncio
@@ -13,9 +13,9 @@ async def test_lifecycle_executes_callbacks__when_app_ends():
     parent.app = app
     parent.callback = callback
 
-    exit()(callback)
+    app_exit()(callback)
 
-    async with exit.lifespan():
+    async with app_exit.lifespan():
         app()
 
     parent.assert_has_calls([call.app(), call.callback()])