Enable ruff security, logging and debugging rules (S, T20, LOG, G, DTZ, ERA, TD, FIX, EXE)

.github/bump_version.py

@@ -3,14 +3,20 @@
 import re
 import sys
 from pathlib import Path
+from typing import TextIO
+
+
+def _report(message: str, *, file: TextIO | None = None) -> None:
+    """Print one line of this command's output, to stdout unless ``file`` is given."""
+    print(message, file=file)
 
 
 def get_current_version(pyproject_path: Path) -> str:
     """Return the version in pyproject.toml, exiting with status 1 if it is absent."""
     text = pyproject_path.read_text()
     match = re.search(r'^version\s*=\s*"(\d+\.\d+\.\d+)"', text, re.MULTILINE)
     if not match:
-        print(
+        _report(
             "Could not find version in pyproject.toml",
             file=sys.stderr,
         )
@@ -28,7 +34,7 @@ def infer_bump(changelog_dir: Path) -> str:
         f for f in changelog_dir.iterdir() if f.is_file() and f.name != ".gitkeep"
     ]
     if not fragments:
-        print("No changelog fragments found", file=sys.stderr)
+        _report("No changelog fragments found", file=sys.stderr)
         sys.exit(1)
 
     categories = {f.suffix.lstrip(".") for f in fragments}
@@ -63,7 +69,7 @@ def update_file(path: Path, old_version: str, new_version: str) -> None:
     )
     if updated != text:
         path.write_text(updated)
-        print(f"  Updated {path}")
+        _report(f"  Updated {path}")
 
 
 def main() -> None:
@@ -76,7 +82,7 @@ def main() -> None:
     bump = infer_bump(changelog_dir)
     new = bump_version(current, bump)
 
-    print(f"Version: {current} -> {new} ({bump})")
+    _report(f"Version: {current} -> {new} ({bump})")
 
     update_file(pyproject, current, new)
 

.github/find-api-model-versions.py

@@ -113,8 +113,8 @@ def find_api_model_versions_and_output_to_github() -> None:
 
     if args.shell:
         for key, value in find_api_model_versions().items():
-            print(f"{key}={shlex.quote(value)}")
+            print(f"{key}={shlex.quote(value)}")  # noqa: T201  # the shell assignments are this command's output
     else:
         find_api_model_versions_and_output_to_github()
-        print("API model versions found and written to GitHub environment.")
+        print("API model versions found and written to GitHub environment.")  # noqa: T201  # the command reports its result
     sys.exit(0)

.github/scripts/check_updates.py

@@ -138,16 +138,16 @@ def main() -> int:
 
     changelog_text = fetch_changelog(args.package)
     if not changelog_text:
-        print(f"Could not fetch changelog for {args.package}.", file=sys.stderr)
+        print(f"Could not fetch changelog for {args.package}.", file=sys.stderr)  # noqa: T201  # the command reports a missing changelog on stderr
         return 0
 
     entries = parse_changelog(changelog_text)
     changes = get_changes_between(entries, args.old_version, args.new_version)
 
     if changes:
-        print(format_changes(changes))
+        print(format_changes(changes))  # noqa: T201  # the formatted changelog is this command's output
     else:
-        print("No changelog entries found between these versions.")
+        print("No changelog entries found between these versions.")  # noqa: T201  # the command reports that nothing changed
 
     return 0
 

.github/scripts/verify_observability_collector.py

@@ -199,12 +199,12 @@ def _raise_for_partial_rejection(
 
 
 def _get_json(url: str, access_token: str) -> dict[str, object] | None:
-    request = Request(
+    request = Request(  # noqa: S310  # callers pass only https Google Cloud API URLs
         url,
         headers={"Authorization": f"Bearer {access_token}"},
     )
     try:
-        with urlopen(request, timeout=15) as response:
+        with urlopen(request, timeout=15) as response:  # noqa: S310  # callers pass only https Google Cloud API URLs
             return json.load(response)
     except HTTPError as error:
         if error.code == 404:
@@ -391,7 +391,7 @@ def main() -> None:
         metric_location=args.metric_location,
         access_token=os.environ["GOOGLE_OAUTH_ACCESS_TOKEN"],
     )
-    print(
+    print(  # noqa: T201  # the command reports the verified ingestion
         "Verified collector ingestion "
         f"(verification_id={probe.verification_id}, trace_id={probe.trace_id})"
     )

AGENTS.md

@@ -122,7 +122,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 - `api.quality.environment-variables-load`: Required environment variables and credentials load in a deployed-like environment. (blocking: CI job `test_env_vars`)
 - `api.quality.mypy-v2`: Shared query models, FastAPI dependencies, and v2 routes and services pass mypy with untyped definitions disallowed. (blocking: mypy)
 - `api.quality.ruff-format`: All Python files are formatted with ruff format. (blocking: ruff format)
-- `api.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (ANN, BLE, D, E4, E7, E9, EM, F, FLY, FURB, I, PIE, PYI, RET, RSE, TC, TRY, UP, W))
+- `api.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W))
 - `api.quality.tests-pass`: The unit, contract and to-refactor test suites pass. (blocking: CI job `test`)
 - `api.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**`)
 

changelog.d/ruff-security-logging.changed.md

@@ -0,0 +1 @@
+Enabled ruff's security, logging and debugging rules (S, T20, LOG, G, DTZ, ERA, TD, FIX, EXE): type-narrowing asserts became explicit checks, command-line scripts keep their printed output with reasons, commented-out code and TODO markers are gone, and every deliberate exception carries a reasoned suppression; error responses and published schemas are unchanged.

changelog.d/ruff-security-logging.fixed.md

@@ -0,0 +1 @@
+Removed the debug prints from the country validator and the report output and simulation routes, so requests no longer write context-free lines to stdout.

dashboard/experiments/gpt4_api_user.py

@@ -5,6 +5,7 @@
 from the resulting economic impact.
 """
 
+import ast
 import json
 import os
 import time
@@ -21,10 +22,10 @@
 
 @st.cache_data
 def get_embeddings_df() -> pd.DataFrame:
-    """Load parameter_embeddings.csv.gz, evaluating each embedding string to a list."""
+    """Load parameter_embeddings.csv.gz, parsing each embedding string to a list."""
     embeddings_df = pd.read_csv("parameter_embeddings.csv.gz", compression="gzip")
     embeddings_df.parameter_embedding = embeddings_df.parameter_embedding.apply(
-        lambda x: eval(x)
+        ast.literal_eval
     )
     return embeddings_df
 
@@ -46,8 +47,9 @@ def embed(prompt: str, engine: str = "text-embedding-ada-002") -> list[float]:
     "Revenue impact of changing the basic rate of income tax to 25%",
 )
 
-# First task: convert to:
-# { 'hmrc.income_tax.rates.uk[0].rate': { '2023-01-01.2024-01-01': 0.25 } }
+# First task: convert the question to a reform that maps each parameter path to
+# its values by period, for example hmrc.income_tax.rates.uk[0].rate set to 0.25
+# for the period 2023-01-01.2024-01-01.
 
 # We're going to do this with GPT4- we'll pass in the question, the related parameter metadata, and the task description.
 
@@ -112,7 +114,7 @@ def get_policyengine_impact(reform: object) -> object:
     payload = {"data": reform}
     url = "https://api.policyengine.org/uk/policy"
 
-    response = requests.post(url, json=payload)
+    response = requests.post(url, json=payload)  # noqa: S113  # a timeout would change this experiment's behaviour; it waits as long as the API takes
 
     # json.result.policy_id
 
@@ -122,13 +124,13 @@ def get_policyengine_impact(reform: object) -> object:
 
     url = f"https://api.policyengine.org/uk/economy/{policy_id}/over/1?time_period=2023&region=uk"
 
-    response = requests.get(url).json()
+    response = requests.get(url).json()  # noqa: S113  # a timeout would change this experiment's behaviour; it waits as long as the API takes
 
     # if 'status' = 'computing', wait 5 seconds and try again until 'status' = 'ok'
 
     while response["status"] == "computing":
         time.sleep(5)
-        response = requests.get(url).json()
+        response = requests.get(url).json()  # noqa: S113  # a timeout would change this experiment's behaviour; it waits as long as the API takes
 
     return response["result"]
 

docs/generated/rules.md

@@ -268,7 +268,7 @@ flowchart LR
 | `api.quality.environment-variables-load` | Required environment variables and credentials load in a deployed-like environment. | mechanical | enforced | yes | CI job `test_env_vars` | `pr.yml:test_env_vars` | — |
 | `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, and v2 routes and services pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — |
 | `api.quality.ruff-format` | All Python files are formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — |
-| `api.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (ANN, BLE, D, E4, E7, E9, EM, F, FLY, FURB, I, PIE, PYI, RET, RSE, TC, TRY, UP, W) | `pr.yml:lint` | — |
+| `api.quality.ruff-lint` | All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. | mechanical | enforced | yes | ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W) | `pr.yml:lint` | — |
 | `api.quality.tests-pass` | The unit, contract and to-refactor test suites pass. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — |
 | `api.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert and need no docstrings, annotations or named constants for magic numbers (S101, D1, ANN, PLR2004); every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, PLR2004, S101 for `**/tests/**` | `pr.yml:lint` | — |
 
@@ -287,7 +287,7 @@ flowchart LR
   e_ruff_format["ruff format"]
   j_CI_pr_yml_lint["CI pr.yml:lint"]
   r_api_quality_ruff_lint("api.quality.ruff-lint (enforced)")
-  e_ruff_check__ANN__BLE__D__E4__E7__E9__EM__F__FLY__FURB__I__PIE__PYI__RET__RSE__TC__TRY__UP__W_["ruff check (ANN, BLE, D, E4, E7, E9, EM, F, FLY, FURB, I, PIE, PYI, RET, RSE, TC, TRY, UP, W)"]
+  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_["ruff check (ANN, BLE, D, DTZ, E4, E7, E9, EM, ERA, EXE, F, FIX, FLY, FURB, G, I, LOG, PIE, PYI, RET, RSE, S, T20, TC, TD, TRY, UP, W)"]
   r_api_quality_tests_pass("api.quality.tests-pass (enforced)")
   e_CI_job_test["CI job test"]
   j_CI_pr_yml_test["CI pr.yml:test"]
@@ -301,8 +301,8 @@ flowchart LR
   e_mypy --> j_CI_pr_yml_typecheck_v2
   r_api_quality_ruff_format --> e_ruff_format
   e_ruff_format --> j_CI_pr_yml_lint
-  r_api_quality_ruff_lint --> e_ruff_check__ANN__BLE__D__E4__E7__E9__EM__F__FLY__FURB__I__PIE__PYI__RET__RSE__TC__TRY__UP__W_
-  e_ruff_check__ANN__BLE__D__E4__E7__E9__EM__F__FLY__FURB__I__PIE__PYI__RET__RSE__TC__TRY__UP__W_ --> j_CI_pr_yml_lint
+  r_api_quality_ruff_lint --> e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_
+  e_ruff_check__ANN__BLE__D__DTZ__E4__E7__E9__EM__ERA__EXE__F__FIX__FLY__FURB__G__I__LOG__PIE__PYI__RET__RSE__S__T20__TC__TD__TRY__UP__W_ --> j_CI_pr_yml_lint
   r_api_quality_tests_pass --> e_CI_job_test
   e_CI_job_test --> j_CI_pr_yml_test
   r_api_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__PLR2004__S101_for____tests___

policyengine_api/api.py

@@ -10,7 +10,7 @@
 def log_timing(message: str) -> None:
     """Print a startup milestone to stderr with the seconds since import began."""
     elapsed = time.time() - start_time
-    print(f"[{elapsed:.2f}s] {message}", file=sys.stderr, flush=True)
+    print(f"[{elapsed:.2f}s] {message}", file=sys.stderr, flush=True)  # noqa: T201  # startup timings go straight to stderr while the app imports
 
 
 log_timing("Basic imports completed")
@@ -31,8 +31,6 @@ def log_timing(message: str) -> None:
 
 log_timing("Caching utilities import completed")
 
-# from werkzeug.middleware.profiler import ProfilerMiddleware
-
 # Endpoints
 from policyengine_api.routes.error_routes import error_bp
 

policyengine_api/asgi_factory.py

@@ -208,13 +208,13 @@ async def add_request_context_and_migration_logging(
                     request.state.policyengine_request_id = request_id
                     _asgi_request_id.reset(context_token)
                     context_token = _asgi_request_id.set(request_id)
-            except Exception:  # noqa: BLE001  # telemetry must never fail the request
+            except Exception:  # noqa: BLE001, S110  # telemetry must never fail the request
                 pass
             try:
                 request_runtime.set_context(
                     request_id=request_id,
                 )
-            except Exception:  # noqa: BLE001  # telemetry must never fail the request
+            except Exception:  # noqa: BLE001, S110  # telemetry must never fail the request
                 pass
 
         def log_native_route(status_code: int) -> None:
@@ -233,7 +233,7 @@ def log_native_route(status_code: int) -> None:
                     ),
                     route_impl=RouteImplementation.FASTAPI_NATIVE,
                 )
-            except Exception:  # noqa: BLE001  # migration logging must never fail the request
+            except Exception:  # noqa: BLE001, S110  # migration logging must never fail the request
                 pass
 
         def finish_native_route(
@@ -245,18 +245,18 @@ def finish_native_route(
             resolved_route = getattr(request.scope.get("route"), "path", initial_route)
             try:
                 request_runtime.update_request_route(resolved_route)
-            except Exception:  # noqa: BLE001  # telemetry must never fail the request
+            except Exception:  # noqa: BLE001, S110  # telemetry must never fail the request
                 pass
             try:
                 request_runtime.update_request_status(status_code)
-            except Exception:  # noqa: BLE001  # telemetry must never fail the request
+            except Exception:  # noqa: BLE001, S110  # telemetry must never fail the request
                 pass
             try:
                 request_runtime.end_request(
                     status_code=status_code,
                     error=error,
                 )
-            except Exception:  # noqa: BLE001  # telemetry must never fail the request
+            except Exception:  # noqa: BLE001, S110  # telemetry must never fail the request
                 pass
 
         try:
@@ -280,13 +280,13 @@ def finish_native_route(
                     request_runtime.set_context(
                         observability_id=response_observability_id
                     )
-                except Exception:  # noqa: BLE001  # telemetry must never fail the request
+                except Exception:  # noqa: BLE001, S110  # telemetry must never fail the request
                     pass
             if native_request:
                 try:
                     for name, value in request_runtime.response_headers().items():
                         response.headers[name] = value
-                except Exception:  # noqa: BLE001  # telemetry must never fail the request
+                except Exception:  # noqa: BLE001, S110  # telemetry must never fail the request
                     pass
             _apply_request_id_header(response, request_id)
             if response_observability_id is not None:

policyengine_api/constants.py

@@ -18,7 +18,7 @@
 UPDATE = "UPDATE"
 LIST = "LIST"
 VERSION = "3.38.1"
-CURRENT_YEAR = str(datetime.now().year)
+CURRENT_YEAR = str(datetime.now().year)  # noqa: DTZ005  # the default year is the host's local calendar year
 COUNTRIES = ("uk", "us", "ca", "ng", "il")
 COUNTRY_PACKAGE_NAMES = (
     "policyengine_uk",
@@ -220,8 +220,8 @@ def _build_runtime_cache_version(
             schema_version,
         )
     )
-    digest = hashlib.sha1(raw.encode("utf-8")).hexdigest()[:digest_length]
-    return f"{prefix}{schema_version}{digest}"
+    digest = hashlib.sha1(raw.encode("utf-8"), usedforsecurity=False).hexdigest()
+    return f"{prefix}{schema_version}{digest[:digest_length]}"
 
 
 def get_economy_impact_cache_version(

policyengine_api/data/v2/catalog/extraction.py

@@ -179,7 +179,7 @@ def normalize_json_value(value: object) -> JSONValue:
 
 def _aware_datetime(value: object, *, field_name: str) -> datetime:
     if isinstance(value, date) and not isinstance(value, datetime):
-        value = datetime(value.year, value.month, value.day)
+        value = datetime(value.year, value.month, value.day, tzinfo=UTC)
     if not isinstance(value, datetime):
         msg = f"{field_name} must be a date or datetime"
         raise CatalogExtractionError(msg)

policyengine_api/data/v2/catalog/initialization.py

@@ -79,9 +79,9 @@ def main() -> int:
     try:
         evidence = initialize_catalog()
     except Exception as error:  # noqa: BLE001  # command must return safe evidence
-        print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr)
+        print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr)  # noqa: T201  # the command's JSON evidence is its output
         return 1
-    print(json.dumps(evidence.as_dict(), sort_keys=True))
+    print(json.dumps(evidence.as_dict(), sort_keys=True))  # noqa: T201  # the command's JSON evidence is its output
     return 0
 
 

policyengine_api/data/v2/household_migration_qualification.py

@@ -262,9 +262,9 @@ def main() -> int:
     try:
         evidence = qualify_household_migration_if_pending()
     except Exception as error:  # noqa: BLE001  # command must emit safe evidence
-        print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr)
+        print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr)  # noqa: T201  # the command's JSON evidence is its output
         return 1
-    print(json.dumps(evidence.as_dict(), sort_keys=True))
+    print(json.dumps(evidence.as_dict(), sort_keys=True))  # noqa: T201  # the command's JSON evidence is its output
     return 0
 
 

policyengine_api/data/v2/policy_migration_qualification.py

@@ -247,9 +247,9 @@ def main() -> int:
     try:
         evidence = qualify_policy_migration_if_pending()
     except Exception as error:  # noqa: BLE001  # command must emit safe evidence
-        print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr)
+        print(json.dumps(_error_payload(error), sort_keys=True), file=sys.stderr)  # noqa: T201  # the command's JSON evidence is its output
         return 1
-    print(json.dumps(evidence.as_dict(), sort_keys=True))
+    print(json.dumps(evidence.as_dict(), sort_keys=True))  # noqa: T201  # the command's JSON evidence is its output
     return 0
 
 

policyengine_api/data/v2/settings.py

@@ -20,7 +20,7 @@
     from collections.abc import Callable, Mapping
 
 V2_RUNTIME_DATABASE_URL = "V2_RUNTIME_DATABASE_URL"
-V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE = "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE"
+V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE = "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE"  # noqa: S105  # an environment variable name, not a secret
 V2_MIGRATION_DATABASE_URL = "V2_MIGRATION_DATABASE_URL"
 V2_DATA_WRITE_DATABASE_URL = "V2_DATA_WRITE_DATABASE_URL"
 V2_SUPABASE_PROJECT_REF = "V2_SUPABASE_PROJECT_REF"

policyengine_api/fastapi_routes/v2/metadata/common.py

@@ -102,5 +102,5 @@ def read_resource(
         if reader is not None:
             try:
                 reader.close()
-            except Exception:  # noqa: BLE001  # best-effort close must not replace the typed response
+            except Exception:  # noqa: BLE001, S110  # best-effort close must not replace the typed response
                 pass