From e41544180bc5ccbd891dc21ac5013618a05c0136 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:01:18 +0400 Subject: [PATCH 01/17] Exempt the rules tool's unittest tests from pytest-style rules The shared rules tool needs only the standard library, so its tests use unittest on purpose. Registered as api.quality.unittest-ruff-exemptions. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 56 ++++++----------------------------------- docs/generated/rules.md | 9 +++++-- pyproject.toml | 3 +++ rules/registry.toml | 19 ++++++++++++++ 4 files changed, 37 insertions(+), 50 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7cf1033eb..b980d9a53 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,66 +70,33 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### API v2 layer boundaries +- Checked: `api.v2.connectors-mutations-no-reads`, `api.v2.connectors-reads-no-mutations`, `api.v2.crud-checks-cover-every-resource`, `api.v2.session-modules-no-queries`, `api.v2.user-services-no-sql`, `api.v2.validators-transformations-no-db` - `api.routes.shared-query-parameters`: Routes that accept query parameters define them through the shared typed query models, never with a second manual parser. (doc-only, not checked: follow it yourself) -- `api.v2.connectors-mutations-no-reads`: v2 mutation connectors (creates, updates, deletes) never issue select or get reads. (blocking: pytest `tests/unit/v2/test_data_crud_boundaries.py`) - `api.v2.connectors-never-commit`: v2 connectors receive a session from the service and never open, commit, roll back or close it. (doc-only, not checked: follow it yourself) - `api.v2.connectors-no-cross-calls`: v2 connector modules never call one another to sequence a workflow; the service does the sequencing. (doc-only, not checked: follow it yourself) -- `api.v2.connectors-reads-no-mutations`: v2 read connectors never insert, add or delete rows. (blocking: pytest `tests/unit/v2/test_data_crud_boundaries.py`) -- `api.v2.crud-checks-cover-every-resource`: The v2 CRUD boundary checks find modules by filename, so every resource package is covered, and every connector module is named as a create, read, update or delete module. (blocking: pytest `tests/unit/v2/test_data_crud_boundaries.py`) - `api.v2.routes-no-sessions-or-sql`: v2 route modules never open database sessions or construct SQL; they call one service method and map typed failures to HTTP. (doc-only, not checked: follow it yourself) - `api.v2.services-no-sql`: v2 services sequence work but never construct SQL expressions, call Session.exec, or define HTTP response models. (doc-only, not checked: follow it yourself) -- `api.v2.session-modules-no-queries`: v2 database_session modules own session lifetime only and never build select, insert, update, delete or exec calls. (blocking: pytest `tests/unit/v2/test_data_crud_boundaries.py`) -- `api.v2.user-services-no-sql`: v2 user services (identity, accounts, legacy references and user associations) never build or run SQL themselves or import SQLAlchemy; their connectors do. (blocking: pytest `tests/unit/v2/test_data_crud_boundaries.py`) -- `api.v2.validators-transformations-no-db`: v2 validators and transformations never import SQLAlchemy or SQLModel query or session APIs. (blocking: pytest `tests/unit/v2/test_data_crud_boundaries.py`) ### Runtime persistence and route boundaries -- `api.data.runtime-sql-confined`: Raw SQL is confined to the data-access layer; runtime modules and report orchestration use sessions and mapped models. (blocking: pytest `tests/unit/data/test_runtime_sql_boundaries.py`) -- `api.legacy.removed-layers-stay-removed`: Removed legacy persistence layers, the temporary DAO module and the legacy endpoints package stay removed. (blocking: pytest `tests/unit/data/test_runtime_sql_boundaries.py`, `tests/unit/routes/test_flask_route_boundaries.py`, `tests/unit/services/test_orm_service_boundaries.py`) -- `api.routes.flask-blueprints-own-urls`: Legacy Flask URLs are owned by blueprints, and the Flask app assembles blueprints without legacy endpoint wiring. (blocking: pytest `tests/unit/routes/test_flask_route_boundaries.py`) -- `api.routes.no-domain-logic-in-http`: Flask routes delegate domain processing (economy comparison, calculation) to services instead of doing it in the HTTP layer. (blocking: pytest `tests/unit/routes/test_flask_route_boundaries.py`) -- `api.routes.no-inline-json-errors`: Flask routes never build JSON error responses inline; they use the shared error handling. (blocking: pytest `tests/unit/routes/test_flask_route_boundaries.py`) -- `api.services.no-direct-queries`: Services never issue database queries or open database connections directly; they delegate persistence to the data-access layer. (blocking: pytest `tests/unit/services/test_local_data_service_boundaries.py`, `tests/unit/services/test_orchestration_service_database_boundaries.py`, `tests/unit/services/test_orm_service_boundaries.py`, `tests/unit/services/test_run_and_spec_service_database_boundaries.py`) -- `api.services.session-ownership`: Migrated services use sessions and mapped models; their public methods never accept persistence objects, and routes and the economy service never manage sessions. (blocking: pytest `tests/unit/services/test_local_data_service_boundaries.py`, `tests/unit/services/test_orm_service_boundaries.py`) +- Checked: `api.data.runtime-sql-confined`, `api.legacy.removed-layers-stay-removed`, `api.routes.flask-blueprints-own-urls`, `api.routes.no-domain-logic-in-http`, `api.routes.no-inline-json-errors`, `api.services.no-direct-queries`, `api.services.session-ownership` ### Import and startup side effects -- `api.catalog.initializer-not-at-startup`: Application startup modules never import the metadata catalog initializer. (blocking: pytest `tests/unit/v2/test_catalog_initialization.py`) -- `api.startup.fail-closed-configuration`: Default Cloud SQL startup needs no Supabase configuration, and a selected v2 runtime without its database URL fails closed. (blocking: pytest `tests/unit/v2/test_import_side_effects.py`) -- `api.startup.no-import-side-effects`: Importing v2 modules, starting up and serving a request open no network connections and create no files or runtime SQLite databases. (blocking: pytest `tests/unit/v2/test_import_side_effects.py`) -- `api.startup.no-runtime-sqlite-modules`: Runtime SQLite modules are absent from the production package. (blocking: pytest `tests/unit/v2/test_import_side_effects.py`) +- Checked: `api.catalog.initializer-not-at-startup`, `api.startup.fail-closed-configuration`, `api.startup.no-import-side-effects`, `api.startup.no-runtime-sqlite-modules` ### Data source isolation -- `api.catalog.extractor-import-isolation`: The catalog extractor never imports PolicyEngine core, country packages or v1 metadata directly. (blocking: pytest `tests/unit/v2/test_catalog_extraction.py`) -- `api.metadata.read-modules-isolated`: v2 metadata read modules never import PolicyEngine packages or v1 metadata sources. (blocking: pytest `tests/unit/v2/test_metadata_service.py`) -- `api.reports.v1-persistence-only`: The existing report route and service import only v1 persistence. (blocking: pytest `tests/unit/v2/test_report_v1_runtime_selection.py`) -- `api.stage12.comparison-tables-isolated`: Serving routes and services never read the Stage 12 comparison tables, and production tables never depend on them. (blocking: pytest `tests/unit/v2/test_stage12_comparison_isolation.py`) -- `api.stage12.retirement-owned`: Retiring the Stage 12 comparison schema stays owned by a named follow-up change in the Stage 12 worker contract. (blocking: pytest `tests/unit/v2/test_stage12_comparison_isolation.py`) -- `api.v2.single-orm-layer`: v2 models never create a parallel SQLAlchemy ORM layer alongside SQLModel. (blocking: pytest `tests/unit/v2/test_model_persistence.py`) +- Checked: `api.catalog.extractor-import-isolation`, `api.metadata.read-modules-isolated`, `api.reports.v1-persistence-only`, `api.stage12.comparison-tables-isolated`, `api.stage12.retirement-owned`, `api.v2.single-orm-layer` ### Migration and schema +- Checked: `api.migration.alembic-v1-qualified`, `api.migration.alembic-v2-qualified`, `api.migration.bundle-supported-by-simulation-api`, `api.migration.contracts-consistent`, `api.migration.cross-database-integration` - `api.migration.alembic-autogenerated-only`: Alembic revisions are autogenerated against an explicitly selected v1 or v2 configuration; never hand-written DDL or data operations. (doc-only, not checked: follow it yourself) -- `api.migration.alembic-v1-qualified`: v1 Alembic revisions qualify against a disposable MySQL database. (blocking: CI job `alembic-v1-check`) -- `api.migration.alembic-v2-qualified`: v2 Alembic revisions qualify against a disposable Postgres database, and alembic check finds no schema drift. (blocking: CI job `alembic-v2-check`) -- `api.migration.bundle-supported-by-simulation-api`: A PolicyEngine bundle version change is supported by the simulation API before it merges. (blocking: CI job `ensure-policyengine-bundle-supported-by-simulation-api`) -- `api.migration.contracts-consistent`: Migration route groups and workflow contracts are internally consistent, and their generated docs are current. (blocking: quality guard `migration-contracts`) -- `api.migration.cross-database-integration`: Cross-database integration tests pass against real MySQL and Postgres. (blocking: CI job `v2-integration-check`) ### Code quality and types -- `api.quality.alembic-revisions-unused-imports`: Ruff doesn't lint Alembic revision files (lint.exclude in pyproject.toml), because they must stay exactly as Alembic generates them. (advisory: nothing, not blocking) -- `api.quality.ci-cd-scripts-ruff-exemptions`: CI/CD scripts under .github may print their results, fetch fixed URLs and import the lazily loaded packages at module level (T201, S310, TID253); every other ruff rule applies to them. (blocking: ruff per-file ignore of S310, T201, TID253 for `.github/**`) -- `api.quality.command-line-modules-ruff-exemptions`: Application modules that also run as command-line programs (if __name__ == "__main__") may print their output (T201); every other ruff rule applies to them. (blocking: ruff per-file ignore of T201 for `policyengine_api/{data/v2/catalog/initialization,data/v2/household_migration_qualification,data/v2/policy_migration_qualification,data/v2/user_association_migration_qualification,worker_spm_release}.py`) -- `api.quality.command-line-scripts-ruff-exemptions`: Command-line scripts under scripts/ may print their output, run other programs, fetch fixed URLs, stamp today's date, pick random samples and ignore a failed best-effort step (T201, S603, S607, S310, DTZ011, S311, S110); every other ruff rule applies to them. (blocking: ruff per-file ignore of DTZ011, S110, S310, S311, S603, S607, T201 for `scripts/**`) -- `api.quality.environment-variables-load`: Required environment variables and credentials load in a deployed-like environment. (blocking: CI job `test_env_vars`) -- `api.quality.mypy-v2`: Shared query models, FastAPI dependencies, v2 routes and services, and the user sign-in and account-deletion libraries pass mypy with untyped definitions disallowed. (blocking: mypy) -- `api.quality.ruff-format`: All Python files are formatted with ruff format. (blocking: ruff format) -- `api.quality.ruff-lint`: All Python code passes ruff check with every rule family selected in pyproject.toml, with no ratchet; families are added one themed group at a time until the selection is ALL. (blocking: ruff check (A, ANN, BLE, D, DTZ, E, E4, E7, E9, EM, ERA, EXE, F, FAST, FIX, FLY, FURB, G, I, INP, LOG, N, PIE, PLC, PYI, RET, RSE, S, SLF, T20, TC, TD, TID253, TRY, UP, W)) -- `api.quality.runtime-evaluated-models-resolve`: Every package model is one ruff treats as runtime-evaluated (a base resolves, through the import in its own module, to a name in runtime-evaluated-base-classes, or is a same-module class that does), and every Pydantic model and SQLAlchemy mapper resolves its annotations once the package is imported. (blocking: pytest `tests/unit/test_runtime_evaluated_annotations.py`) -- `api.quality.tests-pass`: The unit, contract and to-refactor test suites pass. (blocking: CI job `test`) -- `api.quality.tests-ruff-exemptions`: Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also use fake credentials, naive datetimes, subprocess calls to repository scripts, SQL built from strings and prints (S105, S106, DTZ001, DTZ011, S603, S607, S608, T201), and may import the lazily loaded packages at module level (TID253). Every other ruff rule applies to tests. (blocking: ruff per-file ignore of ANN, D1, DTZ001, DTZ011, INP001, PLC0415, PLR2004, S101, S105, S106, S603, S607, S608, SLF001, T201, TID253 for `**/tests/**`) +- Checked: `api.quality.alembic-revisions-unused-imports`, `api.quality.ci-cd-scripts-ruff-exemptions`, `api.quality.command-line-modules-ruff-exemptions`, `api.quality.command-line-scripts-ruff-exemptions`, `api.quality.environment-variables-load`, `api.quality.mypy-v2`, `api.quality.ruff-format`, `api.quality.ruff-lint`, `api.quality.runtime-evaluated-models-resolve`, `api.quality.tests-pass`, `api.quality.tests-ruff-exemptions`, `api.quality.unittest-ruff-exemptions` ### Observability @@ -138,21 +105,14 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### CI workflows and scripts -- `api.ci.multiline-run-is-script`: A GitHub Actions run: block holds one line of code and no chained commands (&&, ||, ;); anything more lives in a tested script under .github/scripts/, called from a single-line step. (blocking: quality guard `ci-scripts`) -- `api.ci.shell-scripts-pass-shellcheck`: Every shell script under .github/scripts/ passes shellcheck. (blocking: quality guard `ci-scripts`) -- `api.ci.workflow-scripts-tested`: Every shell script a workflow calls is run by at least one test. (blocking: quality guard `ci-scripts`) +- Checked: `api.ci.multiline-run-is-script`, `api.ci.shell-scripts-pass-shellcheck`, `api.ci.workflow-scripts-tested` ### Process and repository hygiene -- `api.agents.adapters-point-to-agents-md`: CLAUDE.md and .github/copilot-instructions.md only point to AGENTS.md; skills and rules are listed in AGENTS.md alone. (blocking: quality guard `rules-registry`) +- Checked: `api.agents.adapters-point-to-agents-md`, `api.process.changelog-fragment`, `api.process.cloud-run-image-builds`, `api.process.docker-image-builds`, `api.process.rules-registry`, `api.process.stacked-prs-full-ci-at-bottom` - `api.process.agents-never-bypass`: Agents never bypass branch rules, even with admin credentials: no gh pr merge --admin and no bypass option; a blocked merge is reported to a human. (doc-only, not checked: follow it yourself) -- `api.process.changelog-fragment`: Every PR adds a changelog.d fragment. (blocking: CI job `check-changelog`) -- `api.process.cloud-run-image-builds`: The Cloud Run container image builds. (blocking: CI job `test_cloud_run_container_builds`) -- `api.process.docker-image-builds`: The API Docker image builds. (blocking: CI job `test_container_builds`) - `api.process.makefile-phony`: Command-only Makefile targets are declared in .PHONY, and .PHONY stays in sync when targets are renamed or removed. (doc-only, not checked: follow it yourself) - `api.process.pr-tracking-issue`: Every PR has a tracking issue in this repository, and the first line of its description is Fixes #ISSUE_NUMBER. (doc-only, not checked: follow it yourself) - `api.process.rule-machinery-code-owned`: Changes to the rule machinery (rules/, scripts/rules/, scripts/guards/, scripts/run_quality_guards.py and CODEOWNERS) need approval from a core developer other than the author. (doc-only, not checked: follow it yourself) -- `api.process.rules-registry`: Every rule is registered in rules/registry.toml, is enforced where it claims to be, and its rendered views are current. (blocking: quality guard `rules-registry`) -- `api.process.stacked-prs-full-ci-at-bottom`: In a stack of pull requests, only the lowest unmerged one runs the full PR workflow. Every pr.yml job is needed by the CI complete gate, which fails on the pull requests above it and otherwise passes only when every job passed, so a stack merges bottom-up and nothing merges untested. (blocking: pytest `tests/unit/test_ci_complete_gate.py`) diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 849cff093..0b415bed7 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-api` is listed here with how it is enforce | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 47 | 0 | 1 | 12 | 60 | +| mechanical | 48 | 0 | 1 | 12 | 61 | | metric | 0 | 0 | 0 | 0 | 0 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 47 of 60 rules. +**Blocking:** 48 of 61 rules. ## Gaps @@ -279,6 +279,7 @@ flowchart LR | `api.quality.runtime-evaluated-models-resolve` | Every package model is one ruff treats as runtime-evaluated (a base resolves, through the import in its own module, to a name in runtime-evaluated-base-classes, or is a same-module class that does), and every Pydantic model and SQLAlchemy mapper resolves its annotations once the package is imported. | mechanical | enforced | yes | pytest `tests/unit/test_runtime_evaluated_annotations.py` | `pr.yml:test` | **unproven** | | `api.quality.tests-pass` | The unit, contract and to-refactor test suites pass. | mechanical | enforced | yes | CI job `test` | `pr.yml:test` | — | | `api.quality.tests-ruff-exemptions` | Test files may use pytest's bare assert, need no docstrings, annotations or named constants for magic numbers, and may reach into private members, live outside packages and import inside a test (S101, D1, ANN, PLR2004, SLF001, INP001, PLC0415). They may also use fake credentials, naive datetimes, subprocess calls to repository scripts, SQL built from strings and prints (S105, S106, DTZ001, DTZ011, S603, S607, S608, T201), and may import the lazily loaded packages at module level (TID253). Every other ruff rule applies to tests. | mechanical | enforced | yes | ruff per-file ignore of ANN, D1, DTZ001, DTZ011, INP001, PLC0415, PLR2004, S101, S105, S106, S603, S607, S608, SLF001, T201, TID253 for `**/tests/**` | `pr.yml:lint` | — | +| `api.quality.unittest-ruff-exemptions` | The shared rules tool's tests use unittest, because the tool needs only the standard library, so ruff's pytest-style rules (PT) don't apply to them. | mechanical | enforced | yes | ruff per-file ignore of PT for `scripts/rules/tests/**` | `pr.yml:lint` | — | ```mermaid flowchart LR @@ -308,6 +309,8 @@ flowchart LR e_CI_job_test["CI job test"] r_api_quality_tests_ruff_exemptions("api.quality.tests-ruff-exemptions (enforced)") e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ011__INP001__PLC0415__PLR2004__S101__S105__S106__S603__S607__S608__SLF001__T201__TID253_for____tests___["ruff per-file ignore of ANN, D1, DTZ001, DTZ011, INP001, PLC0415, PLR2004, S101, S105, S106, S603, S607, S608, SLF001, T201, TID253 for **/tests/**"] + r_api_quality_unittest_ruff_exemptions("api.quality.unittest-ruff-exemptions (enforced)") + e_ruff_per_file_ignore_of_PT_for_scripts_rules_tests___["ruff per-file ignore of PT for scripts/rules/tests/**"] r_api_quality_alembic_revisions_unused_imports -.-> gap_unenforced r_api_quality_ci_cd_scripts_ruff_exemptions --> e_ruff_per_file_ignore_of_S310__T201__TID253_for__github___ e_ruff_per_file_ignore_of_S310__T201__TID253_for__github___ --> j_CI_pr_yml_lint @@ -329,6 +332,8 @@ flowchart LR e_CI_job_test --> j_CI_pr_yml_test r_api_quality_tests_ruff_exemptions --> e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ011__INP001__PLC0415__PLR2004__S101__S105__S106__S603__S607__S608__SLF001__T201__TID253_for____tests___ e_ruff_per_file_ignore_of_ANN__D1__DTZ001__DTZ011__INP001__PLC0415__PLR2004__S101__S105__S106__S603__S607__S608__SLF001__T201__TID253_for____tests___ --> j_CI_pr_yml_lint + r_api_quality_unittest_ruff_exemptions --> e_ruff_per_file_ignore_of_PT_for_scripts_rules_tests___ + e_ruff_per_file_ignore_of_PT_for_scripts_rules_tests___ --> j_CI_pr_yml_lint ``` ## Observability diff --git a/pyproject.toml b/pyproject.toml index 7e014260b..cea1d82d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,6 +143,9 @@ exclude = ["migrations/*/versions/*.py"] "ANN", "D1", "DTZ001", "DTZ011", "INP001", "PLC0415", "PLR2004", "S101", "S105", "S106", "S603", "S607", "S608", "SLF001", "T201", "TID253", ] +# The shared rules tool needs only the standard library, so its tests use +# unittest on purpose; pytest-style rules don't apply to them. +"scripts/rules/tests/**" = ["PT"] # CI/CD scripts print their results, fetch fixed URLs and may import the # lazily loaded packages at module level. ".github/**" = ["S310", "T201", "TID253"] diff --git a/rules/registry.toml b/rules/registry.toml index 33bf32d48..60aab4464 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -823,6 +823,25 @@ workflow = ".github/workflows/pr.yml" job = "lint" invocation = "ruff check ." + +[[rule]] +id = "api.quality.unittest-ruff-exemptions" +category = "quality" +statement = "The shared rules tool's tests use unittest, because the tool needs only the standard library, so ruff's pytest-style rules (PT) don't apply to them." +rationale = "docs/engineering/skills/rules-registry.md#baselines" +scope = ["scripts/rules/tests/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "ruff" +mode = "per-file-ignore" +pattern = "scripts/rules/tests/**" +codes = ["PT"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "lint" +invocation = "ruff check ." [[rule]] id = "api.quality.mypy-v2" category = "quality" From 66e2aefe8a3d82ff696109854241388c136d18bf Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:08:35 +0400 Subject: [PATCH 02/17] Use literals and comprehensions (C4, PERF) Rewrite keyword dict() calls as dict literals, generators passed to set() as set comprehensions and a constant-valued dict comprehension as dict.fromkeys. Replace append loops with list.extend or a comprehension, and the two conditional bundle-merging loops with dict.update. Key order, values and messages are unchanged. Two lists that mypy inferred from their first append are now annotated. Co-Authored-By: Claude Opus 5.5 --- policyengine_api/constants.py | 2 +- policyengine_api/country.py | 228 +++++------ policyengine_api/routes/household_routes.py | 8 +- policyengine_api/routes/policy_routes.py | 66 ++-- .../routes/report_output_routes.py | 30 +- policyengine_api/routes/simulation_routes.py | 30 +- .../routes/user_profile_routes.py | 30 +- .../services/economy_comparison.py | 357 +++++++++--------- policyengine_api/services/economy_service.py | 20 +- scripts/compare_migration_baseline.py | 6 +- scripts/guards/migration_contracts.py | 5 +- scripts/rules/baseline.py | 6 +- scripts/rules/ci_scripts.py | 25 +- scripts/rules/discover.py | 102 ++--- scripts/rules/resolve.py | 10 +- scripts/rules/validate.py | 16 +- tests/unit/test_ci_complete_gate.py | 6 +- tests/unit/test_country.py | 2 +- tests/unit/v2/test_household_routes.py | 4 +- tests/unit/v2/test_policy_routes.py | 2 +- 20 files changed, 484 insertions(+), 471 deletions(-) diff --git a/policyengine_api/constants.py b/policyengine_api/constants.py index 492bbde03..84dc5932d 100644 --- a/policyengine_api/constants.py +++ b/policyengine_api/constants.py @@ -166,7 +166,7 @@ def get_bundle_default_dataset_option(country_id: str) -> dict: } except Exception: # noqa: BLE001 # unreadable package metadata falls back to 0.0.0; import must not fail _dist_versions = {} - COUNTRY_PACKAGE_VERSIONS = {country: "0.0.0" for country in COUNTRIES} + COUNTRY_PACKAGE_VERSIONS = dict.fromkeys(COUNTRIES, "0.0.0") _policyengine_bundle = _load_policyengine_bundle() _bundle_package_versions = _resolve_bundle_package_versions(_policyengine_bundle) diff --git a/policyengine_api/country.py b/policyengine_api/country.py index 94aa6ff6a..06930d330 100644 --- a/policyengine_api/country.py +++ b/policyengine_api/country.py @@ -138,26 +138,26 @@ def __init__(self, country_package_name: str, country_id: str) -> None: def build_metadata(self) -> None: """Build the JSON-safe metadata document served by the metadata endpoints.""" self.metadata = self._json_safe( - dict( - variables=self.build_variables(), - parameters=self.build_parameters(), - entities=self.build_entities(), - variableModules=self.tax_benefit_system.variable_module_metadata, - economy_options=self.build_microsimulation_options(), - current_law_id={ + { + "variables": self.build_variables(), + "parameters": self.build_parameters(), + "entities": self.build_entities(), + "variableModules": self.tax_benefit_system.variable_module_metadata, + "economy_options": self.build_microsimulation_options(), + "current_law_id": { "uk": 1, "us": 2, "ca": 3, "ng": 4, "il": 5, }[self.country_id], - basicInputs=self.tax_benefit_system.basic_inputs, - modelled_policies=self.tax_benefit_system.modelled_policies, - spm=spm_metadata(self.country_id), - version=get_package_version( + "basicInputs": self.tax_benefit_system.basic_inputs, + "modelled_policies": self.tax_benefit_system.modelled_policies, + "spm": spm_metadata(self.country_id), + "version": get_package_version( self.country_package_name.replace("_", "-") ), - ) + } ) def _json_safe( @@ -181,7 +181,7 @@ def build_microsimulation_options(self) -> dict: """Build the regions, time periods and datasets offered for economy runs.""" # Shape: { region: [{ name: "uk", label: "the UK" }], # time_period: [{ name: 2022, label: "2022", ... }] } - options = dict() + options = {} if self.country_id == "uk": constituency_names_path = ( Path(__file__).parent / "data" / "constituencies_2024.csv" @@ -189,41 +189,41 @@ def build_microsimulation_options(self) -> dict: constituency_names = pd.read_csv(constituency_names_path) constituency_names = constituency_names.sort_values("name") region = [ - dict(name="uk", label="the UK", type="national"), - dict(name="country/england", label="England", type="country"), - dict(name="country/scotland", label="Scotland", type="country"), - dict(name="country/wales", label="Wales", type="country"), - dict(name="country/ni", label="Northern Ireland", type="country"), + {"name": "uk", "label": "the UK", "type": "national"}, + {"name": "country/england", "label": "England", "type": "country"}, + {"name": "country/scotland", "label": "Scotland", "type": "country"}, + {"name": "country/wales", "label": "Wales", "type": "country"}, + {"name": "country/ni", "label": "Northern Ireland", "type": "country"}, ] - for i in range(len(constituency_names)): - region.append( - dict( - name=f"constituency/{constituency_names.iloc[i]['name']}", - label=constituency_names.iloc[i]["name"], - type="constituency", - ) - ) + region.extend( + { + "name": f"constituency/{constituency_names.iloc[i]['name']}", + "label": constituency_names.iloc[i]["name"], + "type": "constituency", + } + for i in range(len(constituency_names)) + ) local_authority_names_path = ( Path(__file__).parent / "data" / "local_authorities_2021.csv" ) local_authority_names = pd.read_csv(local_authority_names_path) local_authority_names = local_authority_names.sort_values("name") - for i in range(len(local_authority_names)): - region.append( - dict( - name=f"local_authority/{local_authority_names.iloc[i]['name']}", - label=local_authority_names.iloc[i]["name"], - type="local_authority", - ) - ) + region.extend( + { + "name": f"local_authority/{local_authority_names.iloc[i]['name']}", + "label": local_authority_names.iloc[i]["name"], + "type": "local_authority", + } + for i in range(len(local_authority_names)) + ) time_period = [ - dict(name=2024, label="2024"), - dict(name=2025, label="2025"), - dict(name=2026, label="2026"), - dict(name=2027, label="2027"), - dict(name=2028, label="2028"), - dict(name=2029, label="2029"), - dict(name=2030, label="2030"), + {"name": 2024, "label": "2024"}, + {"name": 2025, "label": "2025"}, + {"name": 2026, "label": "2026"}, + {"name": 2027, "label": "2027"}, + {"name": 2028, "label": "2028"}, + {"name": 2029, "label": "2029"}, + {"name": 2030, "label": "2030"}, ] datasets = [get_bundle_default_dataset_option("uk")] options["region"] = region @@ -231,76 +231,76 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "us": region = [ - dict(name="us", label="the US", type="national"), - dict(name="state/al", label="Alabama", type="state"), - dict(name="state/ak", label="Alaska", type="state"), - dict(name="state/az", label="Arizona", type="state"), - dict(name="state/ar", label="Arkansas", type="state"), - dict(name="state/ca", label="California", type="state"), - dict(name="state/co", label="Colorado", type="state"), - dict(name="state/ct", label="Connecticut", type="state"), - dict(name="state/de", label="Delaware", type="state"), - dict(name="state/dc", label="District of Columbia", type="state"), - dict(name="state/fl", label="Florida", type="state"), - dict(name="state/ga", label="Georgia", type="state"), - dict(name="state/hi", label="Hawaii", type="state"), - dict(name="state/id", label="Idaho", type="state"), - dict(name="state/il", label="Illinois", type="state"), - dict(name="state/in", label="Indiana", type="state"), - dict(name="state/ia", label="Iowa", type="state"), - dict(name="state/ks", label="Kansas", type="state"), - dict(name="state/ky", label="Kentucky", type="state"), - dict(name="state/la", label="Louisiana", type="state"), - dict(name="state/me", label="Maine", type="state"), - dict(name="state/md", label="Maryland", type="state"), - dict(name="state/ma", label="Massachusetts", type="state"), - dict(name="state/mi", label="Michigan", type="state"), - dict(name="state/mn", label="Minnesota", type="state"), - dict(name="state/ms", label="Mississippi", type="state"), - dict(name="state/mo", label="Missouri", type="state"), - dict(name="state/mt", label="Montana", type="state"), - dict(name="state/ne", label="Nebraska", type="state"), - dict(name="state/nv", label="Nevada", type="state"), - dict(name="state/nh", label="New Hampshire", type="state"), - dict(name="state/nj", label="New Jersey", type="state"), - dict(name="state/nm", label="New Mexico", type="state"), - dict(name="state/ny", label="New York", type="state"), - dict(name="state/nc", label="North Carolina", type="state"), - dict(name="state/nd", label="North Dakota", type="state"), - dict(name="state/oh", label="Ohio", type="state"), - dict(name="state/ok", label="Oklahoma", type="state"), - dict(name="state/or", label="Oregon", type="state"), - dict(name="state/pa", label="Pennsylvania", type="state"), - dict(name="state/ri", label="Rhode Island", type="state"), - dict(name="state/sc", label="South Carolina", type="state"), - dict(name="state/sd", label="South Dakota", type="state"), - dict(name="state/tn", label="Tennessee", type="state"), - dict(name="state/tx", label="Texas", type="state"), - dict(name="state/ut", label="Utah", type="state"), - dict(name="state/vt", label="Vermont", type="state"), - dict(name="state/va", label="Virginia", type="state"), - dict(name="state/wa", label="Washington", type="state"), - dict(name="state/wv", label="West Virginia", type="state"), - dict(name="state/wi", label="Wisconsin", type="state"), - dict(name="state/wy", label="Wyoming", type="state"), + {"name": "us", "label": "the US", "type": "national"}, + {"name": "state/al", "label": "Alabama", "type": "state"}, + {"name": "state/ak", "label": "Alaska", "type": "state"}, + {"name": "state/az", "label": "Arizona", "type": "state"}, + {"name": "state/ar", "label": "Arkansas", "type": "state"}, + {"name": "state/ca", "label": "California", "type": "state"}, + {"name": "state/co", "label": "Colorado", "type": "state"}, + {"name": "state/ct", "label": "Connecticut", "type": "state"}, + {"name": "state/de", "label": "Delaware", "type": "state"}, + {"name": "state/dc", "label": "District of Columbia", "type": "state"}, + {"name": "state/fl", "label": "Florida", "type": "state"}, + {"name": "state/ga", "label": "Georgia", "type": "state"}, + {"name": "state/hi", "label": "Hawaii", "type": "state"}, + {"name": "state/id", "label": "Idaho", "type": "state"}, + {"name": "state/il", "label": "Illinois", "type": "state"}, + {"name": "state/in", "label": "Indiana", "type": "state"}, + {"name": "state/ia", "label": "Iowa", "type": "state"}, + {"name": "state/ks", "label": "Kansas", "type": "state"}, + {"name": "state/ky", "label": "Kentucky", "type": "state"}, + {"name": "state/la", "label": "Louisiana", "type": "state"}, + {"name": "state/me", "label": "Maine", "type": "state"}, + {"name": "state/md", "label": "Maryland", "type": "state"}, + {"name": "state/ma", "label": "Massachusetts", "type": "state"}, + {"name": "state/mi", "label": "Michigan", "type": "state"}, + {"name": "state/mn", "label": "Minnesota", "type": "state"}, + {"name": "state/ms", "label": "Mississippi", "type": "state"}, + {"name": "state/mo", "label": "Missouri", "type": "state"}, + {"name": "state/mt", "label": "Montana", "type": "state"}, + {"name": "state/ne", "label": "Nebraska", "type": "state"}, + {"name": "state/nv", "label": "Nevada", "type": "state"}, + {"name": "state/nh", "label": "New Hampshire", "type": "state"}, + {"name": "state/nj", "label": "New Jersey", "type": "state"}, + {"name": "state/nm", "label": "New Mexico", "type": "state"}, + {"name": "state/ny", "label": "New York", "type": "state"}, + {"name": "state/nc", "label": "North Carolina", "type": "state"}, + {"name": "state/nd", "label": "North Dakota", "type": "state"}, + {"name": "state/oh", "label": "Ohio", "type": "state"}, + {"name": "state/ok", "label": "Oklahoma", "type": "state"}, + {"name": "state/or", "label": "Oregon", "type": "state"}, + {"name": "state/pa", "label": "Pennsylvania", "type": "state"}, + {"name": "state/ri", "label": "Rhode Island", "type": "state"}, + {"name": "state/sc", "label": "South Carolina", "type": "state"}, + {"name": "state/sd", "label": "South Dakota", "type": "state"}, + {"name": "state/tn", "label": "Tennessee", "type": "state"}, + {"name": "state/tx", "label": "Texas", "type": "state"}, + {"name": "state/ut", "label": "Utah", "type": "state"}, + {"name": "state/vt", "label": "Vermont", "type": "state"}, + {"name": "state/va", "label": "Virginia", "type": "state"}, + {"name": "state/wa", "label": "Washington", "type": "state"}, + {"name": "state/wv", "label": "West Virginia", "type": "state"}, + {"name": "state/wi", "label": "Wisconsin", "type": "state"}, + {"name": "state/wy", "label": "Wyoming", "type": "state"}, ] # Add all 436 congressional districts (435 voting + DC) region.extend(build_congressional_district_metadata()) time_period = [ - dict(name=2035, label="2035"), - dict(name=2034, label="2034"), - dict(name=2033, label="2033"), - dict(name=2032, label="2032"), - dict(name=2031, label="2031"), - dict(name=2030, label="2030"), - dict(name=2029, label="2029"), - dict(name=2028, label="2028"), - dict(name=2027, label="2027"), - dict(name=2026, label="2026"), - dict(name=2025, label="2025"), - dict(name=2024, label="2024"), - dict(name=2023, label="2023"), - dict(name=2022, label="2022"), + {"name": 2035, "label": "2035"}, + {"name": 2034, "label": "2034"}, + {"name": 2033, "label": "2033"}, + {"name": 2032, "label": "2032"}, + {"name": 2031, "label": "2031"}, + {"name": 2030, "label": "2030"}, + {"name": 2029, "label": "2029"}, + {"name": 2028, "label": "2028"}, + {"name": 2027, "label": "2027"}, + {"name": 2026, "label": "2026"}, + {"name": 2025, "label": "2025"}, + {"name": 2024, "label": "2024"}, + {"name": 2023, "label": "2023"}, + {"name": 2022, "label": "2022"}, ] datasets = [get_bundle_default_dataset_option("us")] options["region"] = region @@ -308,10 +308,10 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "ca": region = [ - dict(name="ca", label="Canada"), + {"name": "ca", "label": "Canada"}, ] time_period = [ - dict(name=2023, label="2023"), + {"name": 2023, "label": "2023"}, ] datasets = [{}] options["region"] = region @@ -319,10 +319,10 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "ng": region = [ - dict(name="ng", label="Nigeria"), + {"name": "ng", "label": "Nigeria"}, ] time_period = [ - dict(name=2023, label="2023"), + {"name": 2023, "label": "2023"}, ] datasets = [{}] options["region"] = region @@ -330,10 +330,10 @@ def build_microsimulation_options(self) -> dict: options["datasets"] = datasets elif self.country_id == "il": region = [ - dict(name="il", label="Israel"), + {"name": "il", "label": "Israel"}, ] time_period = [ - dict(name=2023, label="2023"), + {"name": 2023, "label": "2023"}, ] datasets = [{}] options["region"] = region @@ -369,7 +369,7 @@ def build_variables(self) -> dict: } if variable.value_type.__name__ == "Enum": variable_data[variable_name]["possibleValues"] = [ - dict(value=value.name, label=value.value) + {"value": value.name, "label": value.value} for value in variable.possible_values ] variable_data[variable_name]["defaultValue"] = ( diff --git a/policyengine_api/routes/household_routes.py b/policyengine_api/routes/household_routes.py index 26e7b6df0..d7c684c59 100644 --- a/policyengine_api/routes/household_routes.py +++ b/policyengine_api/routes/household_routes.py @@ -145,9 +145,11 @@ def _spm_error_response(error: Exception) -> Response | None: def _calculation_response(calculation: HouseholdCalculationResult) -> dict: - result: dict[str, object] = dict( - status="ok", message=None, result=calculation.household - ) + result: dict[str, object] = { + "status": "ok", + "message": None, + "result": calculation.household, + } if calculation.warnings: result["warnings"] = list(calculation.warnings) if calculation.spm_provenance is not None: diff --git a/policyengine_api/routes/policy_routes.py b/policyengine_api/routes/policy_routes.py index 524506bda..97c23fbac 100644 --- a/policyengine_api/routes/policy_routes.py +++ b/policyengine_api/routes/policy_routes.py @@ -221,13 +221,13 @@ def set_policy(country_id: str) -> Response: except PolicyMirrorUnavailableError: return _policy_mirror_unavailable() - response_body = dict( - status="ok", - message=message, - result=dict( - policy_id=policy_id, - ), - ) + response_body = { + "status": "ok", + "message": message, + "result": { + "policy_id": policy_id, + }, + } code = 200 if is_existing_policy else 201 return Response(json.dumps(response_body), status=code, mimetype="application/json") @@ -265,14 +265,14 @@ def get_policy_search(country_id: str) -> Response: 404, ) - policies = [dict(id=result.id, label=result.label) for result in results] + policies = [{"id": result.id, "label": result.label} for result in results] return Response( json.dumps( - dict( - status="ok", - message="Policies found", - result=policies, - ) + { + "status": "ok", + "message": "Policies found", + "result": policies, + } ), status=200, mimetype="application/json", @@ -365,14 +365,14 @@ def set_user_policy(country_id: str) -> Response: if not creation.created: return Response( json.dumps( - dict( - status="ok", - message=( + { + "status": "ok", + "message": ( f"The reform #{reform_id} / baseline #{baseline_id} pair " f"already exists for user {user_id}" ), - result=dict(id=user_policy.id), - ) + "result": {"id": user_policy.id}, + } ), status=200, mimetype="application/json", @@ -380,11 +380,11 @@ def set_user_policy(country_id: str) -> Response: return Response( json.dumps( - dict( - status="ok", - message="Record created successfully", - result=_serialize_user_policy(user_policy), - ) + { + "status": "ok", + "message": "Record created successfully", + "result": _serialize_user_policy(user_policy), + } ), status=201, mimetype="application/json", @@ -400,11 +400,11 @@ def get_user_policy(country_id: str, user_id: str) -> dict | Response: except ValueError: return _policy_configuration_unavailable() user_policies = user_policy_service.list_user_policies(country_id, user_id) - return dict( - status="ok", - message=None, - result=[_serialize_user_policy(row) for row in user_policies], - ) + return { + "status": "ok", + "message": None, + "result": [_serialize_user_policy(row) for row in user_policies], + } UPDATE_USER_POLICY_ALLOWED_FIELDS = USER_POLICY_MUTABLE_FIELDS @@ -491,11 +491,11 @@ def update_user_policy(country_id: str) -> Response: return Response( json.dumps( - dict( - status="ok", - message="Record updated successfully", - result=dict(id=user_policy_id), - ) + { + "status": "ok", + "message": "Record updated successfully", + "result": {"id": user_policy_id}, + } ), status=200, mimetype="application/json", diff --git a/policyengine_api/routes/report_output_routes.py b/policyengine_api/routes/report_output_routes.py index d1c588fed..0fd282916 100644 --- a/policyengine_api/routes/report_output_routes.py +++ b/policyengine_api/routes/report_output_routes.py @@ -105,11 +105,11 @@ def create_report_output(country_id: str) -> Response: ) status_code = 201 if creation.created else 200 - response_body = dict( - status="ok", - message=message, - result=result, - ) + response_body = { + "status": "ok", + "message": message, + "result": result, + } return Response( json.dumps(response_body), @@ -155,11 +155,11 @@ def get_report_output(country_id: str, report_id: int) -> Response: raise NotFound(msg) result = _serialize_v1_report_output(view) - response_body = dict( - status="ok", - message=None, - result=result, - ) + response_body = { + "status": "ok", + "message": None, + "result": result, + } return Response( json.dumps(response_body), @@ -223,11 +223,11 @@ def update_report_output(country_id: str) -> Response: ) result = _serialize_v1_report_output(view) - response_body = dict( - status="ok", - message="Report output updated successfully", - result=result, - ) + response_body = { + "status": "ok", + "message": "Report output updated successfully", + "result": result, + } return Response( json.dumps(response_body), diff --git a/policyengine_api/routes/simulation_routes.py b/policyengine_api/routes/simulation_routes.py index d3d127d9e..fc55c6a97 100644 --- a/policyengine_api/routes/simulation_routes.py +++ b/policyengine_api/routes/simulation_routes.py @@ -115,11 +115,11 @@ def create_simulation(country_id: str) -> Response: ) status_code = 201 if creation.created else 200 - response_body = dict( - status="ok", - message=message, - result=result, - ) + response_body = { + "status": "ok", + "message": message, + "result": result, + } return Response( json.dumps(response_body), @@ -168,11 +168,11 @@ def get_simulation(country_id: str, simulation_id: int) -> Response: msg = f"Simulation #{simulation_id} not found." raise NotFound(msg) - response_body = dict( - status="ok", - message=None, - result=result, - ) + response_body = { + "status": "ok", + "message": None, + "result": result, + } return Response( json.dumps(response_body), @@ -234,11 +234,11 @@ def update_simulation(country_id: str) -> Response: result = _serialize_v1_simulation(simulation) - response_body = dict( - status="ok", - message="Simulation updated successfully", - result=result, - ) + response_body = { + "status": "ok", + "message": "Simulation updated successfully", + "result": result, + } return Response( json.dumps(response_body), diff --git a/policyengine_api/routes/user_profile_routes.py b/policyengine_api/routes/user_profile_routes.py index 3cc175b5b..2a1e6a00e 100644 --- a/policyengine_api/routes/user_profile_routes.py +++ b/policyengine_api/routes/user_profile_routes.py @@ -54,11 +54,11 @@ def set_user_profile(country_id: str) -> Response: ) result = _serialize_user_profile(profile, include_auth0_id=False) - response = dict( - status="ok", - message="Record created successfully" if created else "Record exists", - result=result, - ) + response = { + "status": "ok", + "message": "Record created successfully" if created else "Record exists", + "result": result, + } return Response( json.dumps(response), status=201 if created else 200, @@ -99,11 +99,11 @@ def get_user_profile(country_id: str) -> Response: msg = "No such user" raise NotFound(msg) - response_body = dict( - status="ok", - message=f"User #{readable_row['user_id']} found successfully", - result=readable_row, - ) + response_body = { + "status": "ok", + "message": f"User #{readable_row['user_id']} found successfully", + "result": readable_row, + } return Response( json.dumps(response_body), @@ -149,11 +149,11 @@ def update_user_profile(country_id: str) -> Response: msg = "No such user id" raise NotFound(msg) - response_body = dict( - status="ok", - message=f"User profile #{user_id} updated successfully", - result=dict(user_id=user_id), - ) + response_body = { + "status": "ok", + "message": f"User profile #{user_id} updated successfully", + "result": {"user_id": user_id}, + } return Response( json.dumps(response_body), diff --git a/policyengine_api/services/economy_comparison.py b/policyengine_api/services/economy_comparison.py index eccf99f22..26c1c362f 100644 --- a/policyengine_api/services/economy_comparison.py +++ b/policyengine_api/services/economy_comparison.py @@ -22,14 +22,14 @@ def budgetary_impact(baseline: dict, reform: dict) -> dict: state_tax_revenue_impact = reform["total_state_tax"] - baseline["total_state_tax"] benefit_spending_impact = reform["total_benefits"] - baseline["total_benefits"] budgetary_impact = tax_revenue_impact - benefit_spending_impact - return dict( - budgetary_impact=budgetary_impact, - tax_revenue_impact=tax_revenue_impact, - state_tax_revenue_impact=state_tax_revenue_impact, - benefit_spending_impact=benefit_spending_impact, - households=sum(baseline["household_weight"]), - baseline_net_income=baseline["total_net_income"], - ) + return { + "budgetary_impact": budgetary_impact, + "tax_revenue_impact": tax_revenue_impact, + "state_tax_revenue_impact": state_tax_revenue_impact, + "benefit_spending_impact": benefit_spending_impact, + "households": sum(baseline["household_weight"]), + "baseline_net_income": baseline["total_net_income"], + } def labor_supply_response(baseline: dict, reform: dict) -> dict: @@ -63,53 +63,53 @@ def labor_supply_response(baseline: dict, reform: dict) -> dict: substitution_lsr_hh = MicroSeries(substitution_lsr_hh, weights=household_weight) income_lsr_hh = MicroSeries(income_lsr_hh, weights=household_weight) - decile_avg = dict( - income=income_lsr_hh.groupby(decile).mean().to_dict(), - substitution=substitution_lsr_hh.groupby(decile).mean().to_dict(), - ) - decile_rel = dict( - income=( + decile_avg = { + "income": income_lsr_hh.groupby(decile).mean().to_dict(), + "substitution": substitution_lsr_hh.groupby(decile).mean().to_dict(), + } + decile_rel = { + "income": ( income_lsr_hh.groupby(decile).sum() / original_earnings.groupby(decile).sum() ).to_dict(), - substitution=( + "substitution": ( substitution_lsr_hh.groupby(decile).sum() / original_earnings.groupby(decile).sum() ).to_dict(), - ) + } - relative_lsr = dict( - income=(income_lsr_hh.sum() / original_earnings.sum()), - substitution=(substitution_lsr_hh.sum() / original_earnings.sum()), - ) + relative_lsr = { + "income": (income_lsr_hh.sum() / original_earnings.sum()), + "substitution": (substitution_lsr_hh.sum() / original_earnings.sum()), + } decile_rel["income"] = {int(k): v for k, v in decile_rel["income"].items() if k > 0} decile_rel["substitution"] = { int(k): v for k, v in decile_rel["substitution"].items() if k > 0 } - hours = dict( - baseline=baseline["weekly_hours"], - reform=reform["weekly_hours"], - change=reform["weekly_hours"] - baseline["weekly_hours"], - income_effect=reform["weekly_hours_income_effect"] + hours = { + "baseline": baseline["weekly_hours"], + "reform": reform["weekly_hours"], + "change": reform["weekly_hours"] - baseline["weekly_hours"], + "income_effect": reform["weekly_hours_income_effect"] - baseline["weekly_hours_income_effect"], - substitution_effect=reform["weekly_hours_substitution_effect"] + "substitution_effect": reform["weekly_hours_substitution_effect"] - baseline["weekly_hours_substitution_effect"], - ) + } - return dict( - substitution_lsr=substitution_lsr, - income_lsr=income_lsr, - relative_lsr=relative_lsr, - total_change=total_change, - revenue_change=revenue_change, - decile=dict( - average=decile_avg, - relative=decile_rel, - ), - hours=hours, - ) + return { + "substitution_lsr": substitution_lsr, + "income_lsr": income_lsr, + "relative_lsr": relative_lsr, + "total_change": total_change, + "revenue_change": revenue_change, + "decile": { + "average": decile_avg, + "relative": decile_rel, + }, + "hours": hours, + } def detailed_budgetary_impact(baseline: dict, reform: dict, country_id: str) -> dict: @@ -121,11 +121,12 @@ def detailed_budgetary_impact(baseline: dict, reform: dict, country_id: str) -> if country_id == "uk": for program in baseline["programs"]: # baseline[programs][program] = total budgetary impact of program - result[program] = dict( - baseline=baseline["programs"][program], - reform=reform["programs"][program], - difference=reform["programs"][program] - baseline["programs"][program], - ) + result[program] = { + "baseline": baseline["programs"][program], + "reform": reform["programs"][program], + "difference": reform["programs"][program] + - baseline["programs"][program], + } return result @@ -164,10 +165,10 @@ def decile_impact(baseline: dict, reform: dict) -> dict: ) rel_decile_dict = rel_income_change_by_decile.to_dict() avg_decile_dict = avg_income_change_by_decile.to_dict() - return dict( - relative={int(k): v for k, v in rel_decile_dict.items()}, - average={int(k): v for k, v in avg_decile_dict.items()}, - ) + return { + "relative": {int(k): v for k, v in rel_decile_dict.items()}, + "average": {int(k): v for k, v in avg_decile_dict.items()}, + } def wealth_decile_impact(baseline: dict, reform: dict) -> dict: @@ -204,10 +205,10 @@ def wealth_decile_impact(baseline: dict, reform: dict) -> dict: ) rel_decile_dict = rel_income_change_by_decile.to_dict() avg_decile_dict = avg_income_change_by_decile.to_dict() - return dict( - relative={int(k): v for k, v in rel_decile_dict.items()}, - average={int(k): v for k, v in avg_decile_dict.items()}, - ) + return { + "relative": {int(k): v for k, v in rel_decile_dict.items()}, + "average": {int(k): v for k, v in avg_decile_dict.items()}, + } def inequality_impact(baseline: dict, reform: dict) -> dict: @@ -221,20 +222,20 @@ def inequality_impact(baseline: dict, reform: dict) -> dict: dict: The impact of the reform on inequality. """ - return dict( - gini=dict( - baseline=baseline["gini"], - reform=reform["gini"], - ), - top_10_pct_share=dict( - baseline=baseline["top_10_percent_share"], - reform=reform["top_10_percent_share"], - ), - top_1_pct_share=dict( - baseline=baseline["top_1_percent_share"], - reform=reform["top_1_percent_share"], - ), - ) + return { + "gini": { + "baseline": baseline["gini"], + "reform": reform["gini"], + }, + "top_10_pct_share": { + "baseline": baseline["top_10_percent_share"], + "reform": reform["top_10_percent_share"], + }, + "top_1_pct_share": { + "baseline": baseline["top_1_percent_share"], + "reform": reform["top_1_percent_share"], + }, + } def poverty_impact(baseline: dict, reform: dict) -> dict: @@ -262,48 +263,48 @@ def poverty_impact(baseline: dict, reform: dict) -> dict: ) age = MicroSeries(baseline["age"]) - poverty = dict( - child=dict( - baseline=float(baseline_poverty[age < 18].mean()), - reform=float(reform_poverty[age < 18].mean()), - ), - adult=dict( - baseline=float(baseline_poverty[(age >= 18) & (age < 65)].mean()), - reform=float(reform_poverty[(age >= 18) & (age < 65)].mean()), - ), - senior=dict( - baseline=float(baseline_poverty[age >= 65].mean()), - reform=float(reform_poverty[age >= 65].mean()), - ), - all=dict( - baseline=float(baseline_poverty.mean()), - reform=float(reform_poverty.mean()), - ), - ) + poverty = { + "child": { + "baseline": float(baseline_poverty[age < 18].mean()), + "reform": float(reform_poverty[age < 18].mean()), + }, + "adult": { + "baseline": float(baseline_poverty[(age >= 18) & (age < 65)].mean()), + "reform": float(reform_poverty[(age >= 18) & (age < 65)].mean()), + }, + "senior": { + "baseline": float(baseline_poverty[age >= 65].mean()), + "reform": float(reform_poverty[age >= 65].mean()), + }, + "all": { + "baseline": float(baseline_poverty.mean()), + "reform": float(reform_poverty.mean()), + }, + } - deep_poverty = dict( - child=dict( - baseline=float(baseline_deep_poverty[age < 18].mean()), - reform=float(reform_deep_poverty[age < 18].mean()), - ), - adult=dict( - baseline=float(baseline_deep_poverty[(age >= 18) & (age < 65)].mean()), - reform=float(reform_deep_poverty[(age >= 18) & (age < 65)].mean()), - ), - senior=dict( - baseline=float(baseline_deep_poverty[age >= 65].mean()), - reform=float(reform_deep_poverty[age >= 65].mean()), - ), - all=dict( - baseline=float(baseline_deep_poverty.mean()), - reform=float(reform_deep_poverty.mean()), - ), - ) + deep_poverty = { + "child": { + "baseline": float(baseline_deep_poverty[age < 18].mean()), + "reform": float(reform_deep_poverty[age < 18].mean()), + }, + "adult": { + "baseline": float(baseline_deep_poverty[(age >= 18) & (age < 65)].mean()), + "reform": float(reform_deep_poverty[(age >= 18) & (age < 65)].mean()), + }, + "senior": { + "baseline": float(baseline_deep_poverty[age >= 65].mean()), + "reform": float(reform_deep_poverty[age >= 65].mean()), + }, + "all": { + "baseline": float(baseline_deep_poverty.mean()), + "reform": float(reform_deep_poverty.mean()), + }, + } - return dict( - poverty=poverty, - deep_poverty=deep_poverty, - ) + return { + "poverty": poverty, + "deep_poverty": deep_poverty, + } def compute_income_change( @@ -374,7 +375,7 @@ def intra_decile_impact(baseline: dict, reform: dict) -> dict: outcome_groups[label].append(people_in_proportion) all_outcomes[label] = sum(outcome_groups[label]) / 10 - return dict(deciles=outcome_groups, all=all_outcomes) + return {"deciles": outcome_groups, "all": all_outcomes} def intra_wealth_decile_impact(baseline: dict, reform: dict) -> dict: @@ -433,7 +434,7 @@ def intra_wealth_decile_impact(baseline: dict, reform: dict) -> dict: outcome_groups[label].append(people_in_proportion) all_outcomes[label] = sum(outcome_groups[label]) / 10 - return dict(deciles=outcome_groups, all=all_outcomes) + return {"deciles": outcome_groups, "all": all_outcomes} def poverty_gender_breakdown(baseline: dict, reform: dict) -> dict: @@ -463,32 +464,32 @@ def poverty_gender_breakdown(baseline: dict, reform: dict) -> dict: ) is_male = MicroSeries(baseline["is_male"]) - poverty = dict( - male=dict( - baseline=float(baseline_poverty[is_male].mean()), - reform=float(reform_poverty[is_male].mean()), - ), - female=dict( - baseline=float(baseline_poverty[~is_male].mean()), - reform=float(reform_poverty[~is_male].mean()), - ), - ) + poverty = { + "male": { + "baseline": float(baseline_poverty[is_male].mean()), + "reform": float(reform_poverty[is_male].mean()), + }, + "female": { + "baseline": float(baseline_poverty[~is_male].mean()), + "reform": float(reform_poverty[~is_male].mean()), + }, + } - deep_poverty = dict( - male=dict( - baseline=float(baseline_deep_poverty[is_male].mean()), - reform=float(reform_deep_poverty[is_male].mean()), - ), - female=dict( - baseline=float(baseline_deep_poverty[~is_male].mean()), - reform=float(reform_deep_poverty[~is_male].mean()), - ), - ) + deep_poverty = { + "male": { + "baseline": float(baseline_deep_poverty[is_male].mean()), + "reform": float(reform_deep_poverty[is_male].mean()), + }, + "female": { + "baseline": float(baseline_deep_poverty[~is_male].mean()), + "reform": float(reform_deep_poverty[~is_male].mean()), + }, + } - return dict( - poverty=poverty, - deep_poverty=deep_poverty, - ) + return { + "poverty": poverty, + "deep_poverty": deep_poverty, + } def poverty_racial_breakdown(baseline: dict, reform: dict) -> dict: @@ -512,28 +513,28 @@ def poverty_racial_breakdown(baseline: dict, reform: dict) -> dict: ) race = MicroSeries(baseline["race"]) # Can be WHITE, BLACK, HISPANIC, or OTHER. - poverty = dict( - white=dict( - baseline=float(baseline_poverty[race == "WHITE"].mean()), - reform=float(reform_poverty[race == "WHITE"].mean()), - ), - black=dict( - baseline=float(baseline_poverty[race == "BLACK"].mean()), - reform=float(reform_poverty[race == "BLACK"].mean()), - ), - hispanic=dict( - baseline=float(baseline_poverty[race == "HISPANIC"].mean()), - reform=float(reform_poverty[race == "HISPANIC"].mean()), - ), - other=dict( - baseline=float(baseline_poverty[race == "OTHER"].mean()), - reform=float(reform_poverty[race == "OTHER"].mean()), - ), - ) + poverty = { + "white": { + "baseline": float(baseline_poverty[race == "WHITE"].mean()), + "reform": float(reform_poverty[race == "WHITE"].mean()), + }, + "black": { + "baseline": float(baseline_poverty[race == "BLACK"].mean()), + "reform": float(reform_poverty[race == "BLACK"].mean()), + }, + "hispanic": { + "baseline": float(baseline_poverty[race == "HISPANIC"].mean()), + "reform": float(reform_poverty[race == "HISPANIC"].mean()), + }, + "other": { + "baseline": float(baseline_poverty[race == "OTHER"].mean()), + "reform": float(reform_poverty[race == "OTHER"].mean()), + }, + } - return dict( - poverty=poverty, - ) + return { + "poverty": poverty, + } class UKConstituencyBreakdownByConstituency(BaseModel): @@ -888,30 +889,30 @@ def compare_economic_outputs( wealth_decile_impact_data = {} intra_wealth_decile_impact_data = {} - return dict( - budget=budgetary_impact_data, - detailed_budget=detailed_budgetary_impact_data, - decile=decile_impact_data, - inequality=inequality_impact_data, - poverty=poverty_impact_data, - poverty_by_gender=poverty_by_gender_data, - poverty_by_race=poverty_by_race_data, - intra_decile=intra_decile_impact_data, - wealth_decile=wealth_decile_impact_data, - intra_wealth_decile=intra_wealth_decile_impact_data, - labor_supply_response=labor_supply_response_data, - constituency_impact=constituency_impact_data, - local_authority_impact=local_authority_impact_data, - ) + return { + "budget": budgetary_impact_data, + "detailed_budget": detailed_budgetary_impact_data, + "decile": decile_impact_data, + "inequality": inequality_impact_data, + "poverty": poverty_impact_data, + "poverty_by_gender": poverty_by_gender_data, + "poverty_by_race": poverty_by_race_data, + "intra_decile": intra_decile_impact_data, + "wealth_decile": wealth_decile_impact_data, + "intra_wealth_decile": intra_wealth_decile_impact_data, + "labor_supply_response": labor_supply_response_data, + "constituency_impact": constituency_impact_data, + "local_authority_impact": local_authority_impact_data, + } if baseline.get("type") == "cliff": - return dict( - baseline=dict( - cliff_gap=baseline["cliff_gap"], - cliff_share=baseline["cliff_share"], - ), - reform=dict( - cliff_gap=reform["cliff_gap"], - cliff_share=reform["cliff_share"], - ), - ) + return { + "baseline": { + "cliff_gap": baseline["cliff_gap"], + "cliff_share": baseline["cliff_share"], + }, + "reform": { + "cliff_gap": reform["cliff_gap"], + "cliff_share": reform["cliff_share"], + }, + } return None diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index d14b0a263..bbdf8a84a 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -1676,18 +1676,26 @@ def _with_policyengine_bundle( "dataset": setup_options.dataset, } if isinstance(result.get("policyengine_bundle"), dict): - for key, value in result["policyengine_bundle"].items(): - if value is not None: - bundle[key] = value + bundle.update( + { + key: value + for key, value in result["policyengine_bundle"].items() + if value is not None + } + ) execution_bundle = ( getattr(execution, "policyengine_bundle", None) if execution is not None else None ) if isinstance(execution_bundle, dict): - for key, value in execution_bundle.items(): - if value is not None: - bundle[key] = value + bundle.update( + { + key: value + for key, value in execution_bundle.items() + if value is not None + } + ) response = { **result, "policyengine_bundle": bundle, diff --git a/scripts/compare_migration_baseline.py b/scripts/compare_migration_baseline.py index d3f320526..6451e95f9 100644 --- a/scripts/compare_migration_baseline.py +++ b/scripts/compare_migration_baseline.py @@ -136,8 +136,10 @@ def compare( f"(≤ {p95_limit:.2f}ms)" ) - for name in sorted(set(candidate_stats) - set(baseline_stats)): - lines.append(f"NOTE {name}: not in baseline; not gated") + lines.extend( + f"NOTE {name}: not in baseline; not gated" + for name in sorted(set(candidate_stats) - set(baseline_stats)) + ) return passed, lines diff --git a/scripts/guards/migration_contracts.py b/scripts/guards/migration_contracts.py index 3018357a5..95c65defa 100644 --- a/scripts/guards/migration_contracts.py +++ b/scripts/guards/migration_contracts.py @@ -23,16 +23,13 @@ def _check_unique_values( *, label: str, ) -> list[str]: - violations = [] seen: set[str] = set() duplicates: set[str] = set() for value in values: if value in seen: duplicates.add(value) seen.add(value) - for value in sorted(duplicates): - violations.append(f"duplicate {label}: {value!r}") - return violations + return [f"duplicate {label}: {value!r}" for value in sorted(duplicates)] def _check_route_groups(payload: dict[str, Any]) -> list[str]: diff --git a/scripts/rules/baseline.py b/scripts/rules/baseline.py index 2af9b1ac5..40242a122 100644 --- a/scripts/rules/baseline.py +++ b/scripts/rules/baseline.py @@ -85,6 +85,8 @@ def check_baselines( "explicit and reviewable", ) ) - for rule_id in sorted(current_ids - {rule.id for rule in base.rules}): - notes.append(f"rule {rule_id} is new") + notes.extend( + f"rule {rule_id} is new" + for rule_id in sorted(current_ids - {rule.id for rule in base.rules}) + ) return violations, notes diff --git a/scripts/rules/ci_scripts.py b/scripts/rules/ci_scripts.py index 77fe38324..170bdc700 100644 --- a/scripts/rules/ci_scripts.py +++ b/scripts/rules/ci_scripts.py @@ -177,24 +177,25 @@ def compare(registry: Registry, findings: Findings, root: Path) -> list[Violatio "fix what `shellcheck