Run ruff check in CI with the first group of rule families

.github/bump_version.py

@@ -42,10 +42,9 @@ def bump_version(version: str, bump: str) -> str:
     major, minor, patch = (int(x) for x in version.split("."))
     if bump == "major":
         return f"{major + 1}.0.0"
-    elif bump == "minor":
+    if bump == "minor":
         return f"{major}.{minor + 1}.0"
-    else:
-        return f"{major}.{minor}.{patch + 1}"
+    return f"{major}.{minor}.{patch + 1}"
 
 
 def update_file(path: Path, old_version: str, new_version: str):

.github/find-api-model-versions.py

@@ -5,6 +5,7 @@
 from importlib.metadata import version as distribution_version
 
 from policyengine.bundle import get_current_bundle
+
 from policyengine_api.constants import (
     COUNTRY_PACKAGE_VERSIONS,
     POLICYENGINE_CORE_VERSION,

.github/workflows/pr.yml

@@ -86,6 +86,8 @@ jobs:
         run: bash .github/scripts/install_locked_ruff.sh
       - name: Format check with ruff
         run: ruff format --check .
+      - name: Lint with ruff
+        run: ruff check .
   typecheck-v2:
     name: Type-check API v2
     # Full CI runs only on the lowest unmerged pull request in a stack.

.github/workflows/push.yml

@@ -29,6 +29,8 @@ jobs:
         run: bash .github/scripts/install_locked_ruff.sh
       - name: Format check with ruff
         run: ruff format --check .
+      - name: Lint with ruff
+        run: ruff check .
 
   quality-guards:
     name: Quality guards

AGENTS.md

@@ -118,11 +118,11 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 
 ### Code quality and types
 
-- `api.quality.alembic-revisions-unused-imports`: Alembic revision files are exempt from unused-import checks (F401), because autogenerated revisions import types they may not use. (advisory: ruff per-file ignore of F401 for `migrations/*/versions/*.py`, not blocking)
+- `api.quality.alembic-revisions-unused-imports`: Alembic revision files are exempt from ruff lint rules, because they must stay exactly as Alembic generates them. (advisory: ruff per-file ignore of ALL for `migrations/*/versions/*.py`, not blocking)
 - `api.quality.environment-variables-load`: Required environment variables and credentials load in a deployed-like environment. (blocking: CI job `test_env_vars`)
 - `api.quality.mypy-v2`: Shared query models, FastAPI dependencies, and v2 routes and services pass mypy with untyped definitions disallowed. (blocking: mypy)
 - `api.quality.ruff-format`: All Python files are formatted with ruff format. (blocking: ruff format)
-- `api.quality.ruff-lint`: Changed Python files pass ruff check with the rules selected in pyproject.toml (ruff's defaults, made explicit); agents run it before committing, but CI does not yet. (advisory: ruff check (E4, E7, E9, F), not blocking)
+- `api.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (E4, E7, E9, F, FLY, FURB, I, PIE, PYI, RET, RSE, UP, W))
 - `api.quality.tests-pass`: The unit, contract and to-refactor test suites pass. (blocking: CI job `test`)
 
 ### Observability

changelog.d/ruff-safe-autofixes.changed.md

@@ -0,0 +1 @@
+Ran ruff check in CI with the first group of rule families (imports, pyupgrade and other safe auto-fixes) and fixed every finding.

dashboard/experiments/gpt4_api_user.py

@@ -1,11 +1,12 @@
-import streamlit as st
+import json
+import os
+import time
+
 import openai
-from openai.embeddings_utils import get_embedding, cosine_similarity
 import pandas as pd
-import json
 import requests
-import time
-import os
+import streamlit as st
+from openai.embeddings_utils import cosine_similarity, get_embedding
 
 # Point OpenAI to the API key
 openai.api_key = os.environ["OPENAI_API_KEY"]
@@ -58,20 +59,18 @@ def embed(prompt, engine="text-embedding-ada-002"):
 
 
 def write_prompt(question, relevant_parameters):
-    prompt = f"""Question: {question}
+    return f"""Question: {question}
     Here's some metadata about the parameters that are relevant to your question:
     Relevant parameters: {relevant_parameters}
 
     Task: Convert the question into a reform of the syntax below. If not specified, assume the reform starts on 2023-01-01 and ends on 2024-01-01.
-    
+
 
     Reform syntax you should write in: {{ "parameter_name": {{ "start_date.end_date": value }} }}
     e.g. "basic rate at 25%" -> {{ "hmrc.income_tax.rates.uk[0].rate": {{ "2023-01-01.2024-01-01": 0.25 }} }}
     Reply only with valid JSON.
     """
 
-    return prompt
-
 
 reform_generation_prompt = write_prompt(question, top5)
 
@@ -80,11 +79,10 @@ def write_prompt(question, relevant_parameters):
 
 @st.cache_data
 def get_gpt4_response(prompt):
-    response = openai.ChatCompletion.create(
+    return openai.ChatCompletion.create(
         model="gpt-4",
         messages=[{"role": "user", "content": prompt}],
     )["choices"][0]["message"]["content"]
-    return response
 
 
 response = get_gpt4_response(reform_generation_prompt)
@@ -117,9 +115,7 @@ def get_policyengine_impact(reform):
         time.sleep(5)
         response = requests.get(url).json()
 
-    impact = response["result"]
-
-    return impact
+    return response["result"]
 
 
 # Now, get GPT4 to answer the question with the impact JSON.

docs/generated/rules.md

@@ -8,11 +8,11 @@ Every rule that governs `policyengine-api` is listed here with how it is enforce
 
 | Tier | enforced | ratcheted | advisory | doc-only | Total |
 | --- | ---: | ---: | ---: | ---: | ---: |
-| mechanical | 40 | 0 | 2 | 12 | 54 |
+| mechanical | 41 | 0 | 1 | 12 | 54 |
 | metric | 0 | 0 | 0 | 0 | 0 |
 | judgment | 0 | 0 | 0 | 0 | 0 |
 
-**Blocking:** 40 of 54 rules.
+**Blocking:** 41 of 54 rules.
 
 ## Gaps
 
@@ -33,8 +33,7 @@ Every rule that governs `policyengine-api` is listed here with how it is enforce
 
 ### Advisory (checked but not blocking)
 
-- `api.quality.alembic-revisions-unused-imports`: Alembic revision files are exempt from unused-import checks (F401), because autogenerated revisions import types they may not use.
-- `api.quality.ruff-lint`: Changed Python files pass ruff check with the rules selected in pyproject.toml (ruff's defaults, made explicit); agents run it before committing, but CI does not yet.
+- `api.quality.alembic-revisions-unused-imports`: Alembic revision files are exempt from ruff lint rules, because they must stay exactly as Alembic generates them.
 
 ### Unproven (no proof that the checker rejects a bad example)
 
@@ -265,17 +264,17 @@ flowchart LR
 
 | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof |
 | --- | --- | --- | --- | --- | --- | --- | --- |
-| `api.quality.alembic-revisions-unused-imports` | Alembic revision files are exempt from unused-import checks (F401), because autogenerated revisions import types they may not use. | mechanical | advisory | no | ruff per-file ignore of F401 for `migrations/*/versions/*.py` | — | — |
+| `api.quality.alembic-revisions-unused-imports` | Alembic revision files are exempt from ruff lint rules, because they must stay exactly as Alembic generates them. | mechanical | advisory | no | ruff per-file ignore of ALL for `migrations/*/versions/*.py` | — | — |
 | `api.quality.environment-variables-load` | Required environment variables and credentials load in a deployed-like environment. | mechanical | enforced | yes | CI job `test_env_vars` | `pr.yml:test_env_vars` | — |
 | `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, and v2 routes and services pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — |
 | `api.quality.ruff-format` | All Python files are formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — |
-| `api.quality.ruff-lint` | Changed Python files pass ruff check with the rules selected in pyproject.toml (ruff's defaults, made explicit); agents run it before committing, but CI does not yet. | mechanical | advisory | no | ruff check (E4, E7, E9, F) | — | — |
+| `api.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (E4, E7, E9, F, FLY, FURB, I, PIE, PYI, RET, RSE, UP, W) | `pr.yml:lint` | — |
 | `api.quality.tests-pass` | The unit, contract and to-refactor test suites pass. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — |
 
 ```mermaid
 flowchart LR
   r_api_quality_alembic_revisions_unused_imports("api.quality.alembic-revisions-unused-imports (advisory)")
-  e_ruff_per_file_ignore_of_F401_for_migrations___versions___py["ruff per-file ignore of F401 for migrations/*/versions/*.py"]
+  e_ruff_per_file_ignore_of_ALL_for_migrations___versions___py["ruff per-file ignore of ALL for migrations/*/versions/*.py"]
   local_only>"not run in CI"]
   r_api_quality_environment_variables_load("api.quality.environment-variables-load (enforced)")
   e_CI_job_test_env_vars["CI job test_env_vars"]
@@ -286,21 +285,21 @@ flowchart LR
   r_api_quality_ruff_format("api.quality.ruff-format (enforced)")
   e_ruff_format["ruff format"]
   j_CI_pr_yml_lint["CI pr.yml:lint"]
-  r_api_quality_ruff_lint("api.quality.ruff-lint (advisory)")
-  e_ruff_check__E4__E7__E9__F_["ruff check (E4, E7, E9, F)"]
+  r_api_quality_ruff_lint("api.quality.ruff-lint (enforced)")
+  e_ruff_check__E4__E7__E9__F__FLY__FURB__I__PIE__PYI__RET__RSE__UP__W_["ruff check (E4, E7, E9, F, FLY, FURB, I, PIE, PYI, RET, RSE, UP, W)"]
   r_api_quality_tests_pass("api.quality.tests-pass (enforced)")
   e_CI_job_test["CI job test"]
   j_CI_pr_yml_test["CI pr.yml:test"]
-  r_api_quality_alembic_revisions_unused_imports --> e_ruff_per_file_ignore_of_F401_for_migrations___versions___py
-  e_ruff_per_file_ignore_of_F401_for_migrations___versions___py -.-> local_only
+  r_api_quality_alembic_revisions_unused_imports --> e_ruff_per_file_ignore_of_ALL_for_migrations___versions___py
+  e_ruff_per_file_ignore_of_ALL_for_migrations___versions___py -.-> local_only
   r_api_quality_environment_variables_load --> e_CI_job_test_env_vars
   e_CI_job_test_env_vars --> j_CI_pr_yml_test_env_vars
   r_api_quality_mypy_v2 --> e_mypy
   e_mypy --> j_CI_pr_yml_typecheck_v2
   r_api_quality_ruff_format --> e_ruff_format
   e_ruff_format --> j_CI_pr_yml_lint
-  r_api_quality_ruff_lint --> e_ruff_check__E4__E7__E9__F_
-  e_ruff_check__E4__E7__E9__F_ -.-> local_only
+  r_api_quality_ruff_lint --> e_ruff_check__E4__E7__E9__F__FLY__FURB__I__PIE__PYI__RET__RSE__UP__W_
+  e_ruff_check__E4__E7__E9__F__FLY__FURB__I__PIE__PYI__RET__RSE__UP__W_ --> j_CI_pr_yml_lint
   r_api_quality_tests_pass --> e_CI_job_test
   e_CI_job_test --> j_CI_pr_yml_test
 ```

migrations/v1/env.py

@@ -1,15 +1,14 @@
 """Alembic environment for the API v1 schema."""
 
-from logging.config import fileConfig
 import os
+from logging.config import fileConfig
 
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from sqlalchemy.engine import make_url
 
 from policyengine_api.data.v1_models import V1Base
 
-
 config = context.config
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)

migrations/v2/env.py

@@ -13,7 +13,6 @@
 )
 from policyengine_api.data.v2.models import V2_METADATA
 
-
 config = context.config
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)

policyengine_api/api.py

@@ -3,8 +3,8 @@
 This is the main Flask app for the PolicyEngine API.
 """
 
-import time
 import sys
+import time
 
 start_time = time.time()
 
@@ -16,18 +16,19 @@ def log_timing(message):
 
 log_timing("Basic imports completed")
 
-from flask_cors import CORS
 import flask
+from flask_cors import CORS
 
 log_timing("Flask imports completed")
 
+from policyengine_observability import instrument_flask
+
 from policyengine_api.extensions import cache
 from policyengine_api.migration_logging import register_migration_request_logging
+from policyengine_api.observability import runtime as observability_runtime
 from policyengine_api.observability.identifiers import OBSERVABILITY_ID_HEADER
 from policyengine_api.request_context import REQUEST_ID_HEADER
-from policyengine_api.observability import runtime as observability_runtime
 from policyengine_api.runtime_cache.settings import load_runtime_cache_settings
-from policyengine_observability import instrument_flask
 
 log_timing("Caching utilities import completed")
 
@@ -55,9 +56,9 @@ def log_timing(message):
 from policyengine_api.routes.user_profile_routes import user_profile_bp
 
 log_timing("User profile routes import completed")
-from policyengine_api.routes.simulation_routes import simulation_bp
-from policyengine_api.routes.report_output_routes import report_output_bp
 from policyengine_api.routes.reform_impact_routes import reform_impact_bp
+from policyengine_api.routes.report_output_routes import report_output_bp
+from policyengine_api.routes.simulation_routes import simulation_bp
 from policyengine_api.routes.system_routes import system_bp
 
 log_timing("Initialising API...")

policyengine_api/asgi_factory.py

@@ -2,18 +2,28 @@
 
 from __future__ import annotations
 
+import time
 from collections.abc import Callable
 from contextlib import asynccontextmanager
-import time
 
 from a2wsgi import WSGIMiddleware
 from fastapi import FastAPI, Request
 from fastapi.exception_handlers import request_validation_exception_handler
 from fastapi.exceptions import RequestValidationError
+from policyengine_observability import ObservabilityRuntime
+from starlette.datastructures import MutableHeaders
+from starlette.middleware.cors import CORSMiddleware
+from starlette.middleware.gzip import GZipMiddleware
+from starlette.responses import PlainTextResponse, Response
+from starlette.routing import Match, Mount
+from starlette.types import ASGIApp
+
 from policyengine_api.constants import VERSION
 from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies
-from policyengine_api.fastapi_routes.health import build_core_health_router
-from policyengine_api.fastapi_routes.health import build_readiness_router
+from policyengine_api.fastapi_routes.health import (
+    build_core_health_router,
+    build_readiness_router,
+)
 from policyengine_api.fastapi_routes.metadata import build_metadata_router
 from policyengine_api.fastapi_routes.specification import (
     build_specification_router,
@@ -29,6 +39,10 @@
 )
 from policyengine_api.migration_logging import log_migration_request
 from policyengine_api.observability import get_runtime
+from policyengine_api.observability.identifiers import (
+    OBSERVABILITY_ID_HEADER,
+    normalize_observability_id,
+)
 from policyengine_api.request_context import (
     REQUEST_ID_HEADER,
     _asgi_incoming_observability_id,
@@ -37,17 +51,6 @@
     current_observability_id,
     generate_request_id,
 )
-from policyengine_observability import ObservabilityRuntime
-from policyengine_api.observability.identifiers import (
-    OBSERVABILITY_ID_HEADER,
-    normalize_observability_id,
-)
-from starlette.datastructures import MutableHeaders
-from starlette.middleware.cors import CORSMiddleware
-from starlette.middleware.gzip import GZipMiddleware
-from starlette.responses import PlainTextResponse, Response
-from starlette.routing import Match, Mount
-from starlette.types import ASGIApp
 
 
 def _apply_request_id_header(