Enable ruff structure and import rules (INP, PLC, SLF, N, A, FAST, E)

.github/find-api-model-versions.py

@@ -1,3 +1,4 @@
+# noqa: N999  # update-policyengine-package.sh runs this script by its hyphenated path
 """CI helper that reads the versions pinned by the installed PolicyEngine bundle.
 
 It checks the package and data versions against the API's version constants,

AGENTS.md

@@ -122,9 +122,9 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 - `api.quality.environment-variables-load`: Required environment variables and credentials load in a deployed-like environment. (blocking: CI job `test_env_vars`)
 - `api.quality.mypy-v2`: Shared query models, FastAPI dependencies, and v2 routes and services pass mypy with untyped definitions disallowed. (blocking: mypy)
 - `api.quality.ruff-format`: All Python files are formatted with ruff format. (blocking: ruff format)
-- `api.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W))
+- `api.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W))
 - `api.quality.tests-pass`: The unit, contract and to-refactor test suites pass. (blocking: CI job `test`)
-- `api.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**`)
+- `api.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/tests/**`)
 
 ### Observability
 

changelog.d/ruff-structure-imports.changed.md

@@ -0,0 +1 @@
+Enabled ruff's structure and import rules (INP, PLC, SLF, N, A, FAST, E): long lines are wrapped, FastAPI dependencies are declared with Annotated, deferred imports that avoid heavy loads, cycles or import-time I/O carry a reason, and directories without an __init__.py on purpose are declared to ruff as namespace packages; tests are also exempt from SLF001, INP001 and PLC0415. Published schemas and the HTTP API are unchanged.

dashboard/experiments/gpt4_api_user.py

@@ -51,7 +51,8 @@ def embed(prompt: str, engine: str = "text-embedding-ada-002") -> list[float]:
 # its values by period, for example hmrc.income_tax.rates.uk[0].rate set to 0.25
 # for the period 2023-01-01.2024-01-01.
 
-# We're going to do this with GPT4- we'll pass in the question, the related parameter metadata, and the task description.
+# We're going to do this with GPT4- we'll pass in the question, the related parameter
+# metadata, and the task description.
 
 # First, get related parameters via a search through GPT-embedded parameters.
 
@@ -81,7 +82,7 @@ def write_prompt(question: str, relevant_parameters: object) -> str:
     Reform syntax you should write in: {{ "parameter_name": {{ "start_date.end_date": value }} }}
     e.g. "basic rate at 25%" -> {{ "hmrc.income_tax.rates.uk[0].rate": {{ "2023-01-01.2024-01-01": 0.25 }} }}
     Reply only with valid JSON.
-    """
+    """  # noqa: E501  # the prompt is sent to GPT-4 verbatim; wrapping would change it
 
 
 reform_generation_prompt = write_prompt(question, top5)
@@ -146,7 +147,7 @@ def get_policyengine_impact(reform: object) -> object:
     Task: Write a concise single-sentence response to the question, using only the impact data. If you can't from the data, say it.
 
     Give financial amounts in short-form (e.g. £1bn, 250m, etc.).
-"""
+"""  # noqa: E501  # the prompt is sent to GPT-4 verbatim; wrapping would change it
 
 answer = get_gpt4_response(question_answer_prompt)
 

docs/generated/rules.md

@@ -268,9 +268,9 @@ flowchart LR
 | `api.quality.environment-variables-load` | Required environment variables and credentials load in a deployed-like environment. | mechanical | enforced | yes | CI job `test_env_vars` | `pr.yml:test_env_vars` | — |
 | `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, and v2 routes and services pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — |
 | `api.quality.ruff-format` | All Python files are formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — |
-| `api.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — |
+| `api.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — |
 | `api.quality.tests-pass` | The unit, contract and to-refactor test suites pass. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — |
-| `api.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**` | `pr.yml:lint` | — |
+| `api.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415); every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for `**/tests/**` | `pr.yml:lint` | — |
 
 ```mermaid
 flowchart LR
@@ -287,12 +287,12 @@ flowchart LR
   e_ruff_format["ruff format"]
   j_CI_pr_yml_lint["CI pr.yml:lint"]
   r_api_quality_ruff_lint("api.quality.ruff-lint (enforced)")
-  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_["ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W)"]
+  e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_["ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TRY, UP, W)"]
   r_api_quality_tests_pass("api.quality.tests-pass (enforced)")
   e_CI_job_test["CI job test"]
   j_CI_pr_yml_test["CI pr.yml:test"]
   r_api_quality_tests_ruff_exemptions("api.quality.tests-ruff-exemptions (enforced)")
-  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___["ruff per-file ignore of ANN, D1, PLR2004, S101 for **/tests/**"]
+  e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___["ruff per-file ignore of ANN, D1, INP001, PLC0415, PLR2004, S101, SLF001 for **/tests/**"]
   r_api_quality_alembic_revisions_unused_imports --> e_ruff_per_file_ignore_of_ALL_for_migrations___versions___py
   e_ruff_per_file_ignore_of_ALL_for_migrations___versions___py -.-> local_only
   r_api_quality_environment_variables_load --> e_CI_job_test_env_vars
@@ -301,12 +301,12 @@ flowchart LR
   e_mypy --> j_CI_pr_yml_typecheck_v2
   r_api_quality_ruff_format --> e_ruff_format
   e_ruff_format --> j_CI_pr_yml_lint
-  r_api_quality_ruff_lint --> e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_
-  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint
+  r_api_quality_ruff_lint --> e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_
+  e_ruff_check__A__ANN__BLE__D__DTZ__E__E4__E7__E9__EM__ERA__EXE__F__FAST__FIX__FLY__FURB__G__I__INP__LOG__N__PIE__PLC__PYI__RET__RSE__S__SLF__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint
   r_api_quality_tests_pass --> e_CI_job_test
   e_CI_job_test --> j_CI_pr_yml_test
-  r_api_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___
-  e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___ --> j_CI_pr_yml_lint
+  r_api_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___
+  e_ruff_per_file_ignore_of_ANN__D1__INP001__PLC0415__PLR2004__S101__SLF001_for____tests___ --> j_CI_pr_yml_lint
 ```
 
 ## Observability

policyengine_api/constants.py

@@ -246,13 +246,15 @@ def get_report_output_cache_version(country_id: str) -> str:
     "national",  # National level (e.g., "us")
     "state",  # US states (e.g., "state/ca", "state/ny")
     "place",  # US Census places (e.g., "place/NJ-57000")
-    "congressional_district",  # US congressional districts (e.g., "congressional_district/CA-37")
+    # US congressional districts (e.g., "congressional_district/CA-37")
+    "congressional_district",
 )
 
 UK_REGION_TYPES = (
     "national",  # National level (e.g., "uk")
     "country",  # UK countries (e.g., "country/england", "country/scotland")
-    "constituency",  # UK parliamentary constituencies (e.g., "constituency/Aldershot")
+    # UK parliamentary constituencies (e.g., "constituency/Aldershot")
+    "constituency",
     "local_authority",  # UK local authorities (e.g., "local_authority/Maidstone")
 )
 
@@ -262,11 +264,13 @@ def get_report_output_cache_version(country_id: str) -> str:
     "us": [
         "state/",  # US states (e.g., "state/ca", "state/ny")
         "place/",  # US Census places (e.g., "place/NJ-57000")
-        "congressional_district/",  # US congressional districts (e.g., "congressional_district/CA-37")
+        # US congressional districts (e.g., "congressional_district/CA-37")
+        "congressional_district/",
     ],
     "uk": [
         "country/",  # UK countries (e.g., "country/england", "country/scotland")
-        "constituency/",  # UK parliamentary constituencies (e.g., "constituency/Aldershot")
+        # UK parliamentary constituencies (e.g., "constituency/Aldershot")
+        "constituency/",
         "local_authority/",  # UK local authorities (e.g., "local_authority/Maidstone")
     ],
 }

policyengine_api/country.py

@@ -179,7 +179,8 @@ def _json_safe(
 
     def build_microsimulation_options(self) -> dict:
         """Build the regions, time periods and datasets offered for economy runs."""
-        # { region: [{ name: "uk", label: "the UK" }], time_period: [{ name: 2022, label: "2022", ... }] }
+        # Shape: { region: [{ name: "uk", label: "the UK" }],
+        #   time_period: [{ name: 2022, label: "2022", ... }] }
         options = dict()
         if self.country_id == "uk":
             constituency_names_path = (
@@ -396,7 +397,8 @@ def build_parameters(self) -> dict:
                 }
             elif isinstance(parameter, ParameterScaleBracket):
                 bracket_index = int(parameter.name[parameter.name.index("[") + 1 : -1])
-                # Set the label to 'first bracket' for the first bracket, 'second bracket' for the second, etc.
+                # Set the label to 'first bracket' for the first bracket, 'second
+                # bracket' for the second, etc.
                 bracket_label = f"bracket {bracket_index + 1}"
                 parameter_data[parameter.name] = {
                     "type": "parameterNode",
@@ -681,7 +683,9 @@ def modify_parameters(parameters: ParameterNode) -> ParameterNode:
                 start, end = period.split(".")
                 node_type = type(node.values_list[-1].value)
                 if node_type is int:
-                    node_type = float  # '0' is of type int by default, but usually we want to cast to float.
+                    # '0' is of type int by default, but usually we want to cast to
+                    # float.
+                    node_type = float
                 if node.values_list[-1].value is None:
                     node_type = float
                 node.update(
@@ -692,7 +696,7 @@ def modify_parameters(parameters: ParameterNode) -> ParameterNode:
 
         return parameters
 
-    class reform(Reform):
+    class reform(Reform):  # noqa: N801  # policyengine-core uses the class name as the reform key
         def apply(self) -> None:
             self.modify_parameters(modify_parameters)
 

policyengine_api/data/v2/catalog/extraction.py

@@ -745,8 +745,10 @@ def extract_catalog(
 
 def extract_installed_catalog() -> NormalizedCatalog:
     """Load and normalize the installed PolicyEngine.py certified catalog."""
-    import policyengine
-    from policyengine.bundle import get_current_bundle
+    import policyengine  # noqa: PLC0415  # policyengine loads every country model; only catalog extraction needs it
+    from policyengine.bundle import (  # noqa: PLC0415  # policyengine loads every country model; only catalog extraction needs it
+        get_current_bundle,
+    )
 
     policyengine_version = importlib_metadata.version("policyengine")
     return extract_catalog(

policyengine_api/data/v2/settings.py

@@ -92,7 +92,9 @@ def _required(environ: Mapping[str, str], name: str) -> str:
 @cache
 def _load_secret_from_secret_manager(resource_name: str) -> str:
     """Resolve a v2 runtime URL only when a preview request selects it."""
-    from google.cloud import secretmanager
+    from google.cloud import (  # noqa: PLC0415  # the Google client loads only when a secret resource is configured
+        secretmanager,
+    )
 
     client = secretmanager.SecretManagerServiceClient()
     response = client.access_secret_version(request={"name": resource_name})