From 11521ae6c9e69fb707ae6feb9f50970421348598 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 04:29:29 -0400 Subject: [PATCH 1/5] Build benefit units from concept pointers and execute group encoding (NZ G4) Unit construction (microcosm.frame.unit_construction): - build_benefit_units(person, household, rule) -> (family table, person_family_id). A unit is an adult, their co-resident partner and their dependent children; every other adult is their own unit (method card MC7). Who counts as a dependent child is a declared BenefitUnitRule (format microcosm.benefit_unit_rule.v1): maximum age, an optional employment-based financial-independence test, where a child with no co-resident parent goes (reference person's unit, own unit, or refuse) and what happens to a child whose two co-resident parents are in different units (refuse, or parent 1). rulespec-nz 8dc2507 encodes no dependent-child definition, so the rule is spec data (amendment MC11a). Invalid pointers are refused through validate_concept_tables. Unit ids are the head's person id, the table is sorted by id (a Frame group table requirement), and units carry no weight. - benefit_unit_roles, benefit_unit_attributes, benefit_unit_membership and units_per_household for the unit-attributes node and group encoding. Group encoding (ConceptMapping.encode_groups): - Executes sum_over_members, any_member, reference_member, household_value and allocate_to_reference_unit on a checked GroupMembership; person_role bindings and entities with no membership stay deferred. encode() is unchanged and still defers every group binding. - Executes declared StateBindings: group inputs fed from model state (receipt flags, an assigned area), not concepts. - GroupKnobs: allocation reference_unit | per_adult_share (MC8), zeroed inputs (asset test off, MC10), state-column substitution (area column, MC9) and factors (rent factor). A knob naming an input the call does not produce or cannot change is refused. - New transforms: scaled_sum (a sum, then a fixed scale) and unit_composition (has_partner, has_dependent_child, two_or_more_dependent_children, sole_parent on the built unit). Input closure (microcosm.frame.input_closure, build/nz/axiom_input_closure.json): - Every root input of the four modules the NZ v0 graph binds is closed exactly once against the committed input surface (entitlement 60, rates 28, NZS core 13, AS core 39): 41 concept bindings (one awaiting G3a), 4 state bindings (area flags, non-beneficiary), 2 graph-node inputs (the reg 17/18 bridge values, not computed here), 93 defaults with reasons and citations, all listed for D1. No input is classed engine-optional: axiom-rules-engine 04315d94 has no optional inputs (an absent input that a row reaches fails the request). The closure is for transport frames, so a binding that reads a concept transport drops cannot encode an input there. - Undetermined judgments: takeup.assign@1 refuses code 0 (none can arise at 04315d94 with every input fed); resolve_judgments implements all three actions. Receipt-layer requirements (one main benefit per person, never NZS with a main benefit, a positive income-tested net payment) are declared. - The cash-asset entry awaits G3a's liquid-asset binding; delete its "awaiting" note when G3a lands (the closure check reports it until then). axiom:nz mapping: Accommodation Supplement has_dependent_children (now from the built unit), has_two_or_more_dependent_children, sole_parent, owner_weekly_required_payments ((interest + principal) / 52 on the reference family), owns_premises_and_not_joint_owner_with_resident and relevant_weekly_income; main-benefit rates income inputs. Coverage golden and docs/concept-coverage/axiom-nz.md regenerated (88 inputs fed by a concept). Invariants (Hypothesis property tests): - partition: every person is in exactly one unit; every unit has members - every unit has one head (an adult) and at most one partner - partners share a unit - every dependent child shares a unit with a co-resident parent, or with the household reference person when it has none - units nest in households - sum of family weights = sum over households of household weight x units in the household (family weights inherited through Frame.resolve_weights) - unit ids are deterministic and independent of row order - household_value is constant within a household - allocate_to_reference_unit puts each household amount on exactly the unit holding the reference person and conserves it; per_adult_share conserves it - sum_over_members conserves member totals - the ADR round-trip, idempotence and conservation properties still hold for every mapping (existing suite, unchanged pins) - the closure partitions each bound module's surface exactly Differential tests: encode_groups vs a row-by-row reference with independent roles; unit_composition vs benefit_unit_attributes; golden-08 encoded inputs vs the oracle harness's 28 AS inputs (engine-free); golden-08 through the real AS module vs the oracle's weekly amount within $0.01 (engine-present, skipped without axiom_rules_engine and POPULACE_RULESPEC_NZ). Depends on G2's build/nz/country_package.json: add the row {"path": "axiom_input_closure.json", "kind": "legacy_json", "schema_id": "legacy_json"}. Until then test__given_country_like_directories__then_each_has_a_manifest fails (build/nz/ has no manifest on main). axiom: n/a: microsim-side unit construction, group encoding and input closure; no policy is encoded in a country model. Co-Authored-By: Claude Opus 5.5 --- .../unit-construction-group-encode.added.md | 1 + docs/agent-guide.md | 8 + docs/concept-coverage/axiom-nz.md | 42 +- docs/concept-schema-transport-adr.md | 79 +- .../build/nz/axiom_input_closure.json | 1428 +++++++++++++++++ .../shared/test_nz_axiom_input_closure.py | 408 +++++ ...le_golden_08_accommodation_supplement.json | 180 +++ .../adapters/axiom_concept_mappings/nz.json | 206 ++- .../src/microcosm/frame/concept_mapping.py | 900 ++++++++++- .../src/microcosm/frame/concepts.py | 3 +- .../src/microcosm/frame/input_closure.py | 850 ++++++++++ .../src/microcosm/frame/unit_construction.py | 678 ++++++++ .../engine_free/shared/test_group_encode.py | 1002 ++++++++++++ .../engine_free/shared/test_input_closure.py | 599 +++++++ .../shared/test_unit_construction.py | 488 ++++++ .../golden/concept-coverage/axiom-nz.json | 359 ++++- .../microcosm_frame/concept_engine_frames.py | 9 +- .../microcosm_frame/concept_reference.py | 4 + test_support/microcosm_frame/unit_rules.py | 47 + 19 files changed, 7172 insertions(+), 119 deletions(-) create mode 100644 changelog.d/unit-construction-group-encode.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py create mode 100644 packages/microcosm-build/tests/fixtures/nz/axiom_oracle_golden_08_accommodation_supplement.json create mode 100644 packages/microcosm-frame/src/microcosm/frame/input_closure.py create mode 100644 packages/microcosm-frame/src/microcosm/frame/unit_construction.py create mode 100644 packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py create mode 100644 packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py create mode 100644 packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py create mode 100644 test_support/microcosm_frame/unit_rules.py diff --git a/changelog.d/unit-construction-group-encode.added.md b/changelog.d/unit-construction-group-encode.added.md new file mode 100644 index 0000000000..a26a2ba730 --- /dev/null +++ b/changelog.d/unit-construction-group-encode.added.md @@ -0,0 +1 @@ +Add benefit-unit construction from concept pointers (`microcosm.frame.unit_construction`), group encoding that executes a mapping's group rules, state bindings and scenario knobs on built units (`ConceptMapping.encode_groups`), and Axiom input closures (`microcosm.frame.input_closure`). Extend the `axiom:nz` mapping with the Accommodation Supplement unit-composition flags, owner payments and relevant income, and the main-benefit rates income inputs, and commit the New Zealand input closure (`build/nz/axiom_input_closure.json`), which closes all 140 root inputs of the four bound modules against the committed input surface. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index f006103a36..84a0c58c68 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -276,6 +276,14 @@ Axiom mappings are data (`adapters/axiom_concept_mappings/.json`), checked against the engine-generated input surfaces in `packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/` (regenerate with `tools/refresh_axiom_input_surface.py`; it needs a real Axiom engine build). +Group bindings run through `ConceptMapping.encode_groups` on units from +`microcosm.frame.unit_construction`; `encode` still reports them as deferred. +The Axiom engine fails a request that reaches an input it was not given, so a +country pack that binds Axiom modules closes every root input in an input +closure (`build//axiom_input_closure.json`, read by +`microcosm.frame.input_closure`), and its test checks the closure against the +committed surface. A change that binds, unbinds or moves a mapping input +updates its closure entry too; the closure test fails until the two agree. ## Root journals are history, not state diff --git a/docs/concept-coverage/axiom-nz.md b/docs/concept-coverage/axiom-nz.md index abc2b0f46a..2d5fbe1106 100644 --- a/docs/concept-coverage/axiom-nz.md +++ b/docs/concept-coverage/axiom-nz.md @@ -3,47 +3,47 @@ Engine version `6fe181fc4c65763150fe97501ef3443774e64869`; concept schema `d4cb09218a97`. Generated by `tools/refresh_concept_coverage.py`; do not edit by hand. - Engine inputs: 668 -- Fed by a concept: 79 +- Fed by a concept: 88 - Structural: 0 -- Uncovered: 589 +- Uncovered: 580 - Mapped but missing from the engine: 0 ## Inputs each concept feeds | Concept | Engine inputs | |---|---| -| `fact:person.age` | `accommodation_supplement_has_dependent_children` (Family)
`caregiver_age_years` (Person)
`childcare_assistance_dependent_children_count` (Person)
`community_services_card_dependent_children_count` (Person)
`community_services_card_person_age_years` (Person)
`disability_allowance_dependent_children_count` (Person)
`disability_allowance_person_age_years` (Person)
`disability_allowance_person_is_sole_parent` (Person)
`in_work_tax_credit_allowed_children_count` (Family)
`in_work_tax_credit_person_age` (Family)
`jobseeker_applicant_age` (Person)
`jobseeker_has_dependent_child` (Person)
`jobseeker_support_applicant_age` (Person)
`jobseeker_support_dependent_children_count` (Person)
`nz_super_has_dependent_children` (Person)
`nz_super_person_age` (Person)
`sole_parent_applicant_age` (Person)
`sole_parent_caring_for_dependent_children_under_age_limit` (Person)
`sole_parent_is_mother_or_father_or_child_regarded_as_child` (Person)
`student_allowance_has_children` (Person)
`student_allowance_student_age` (Person)
`supported_living_applicant_age` (Person)
`supported_living_caring_applicant_age` (Person)
`supported_living_caring_applicant_has_dependent_child` (Person)
`supported_living_payment_applicant_age` (Person)
`supported_living_payment_dependent_children_count` (Person)
`wff_dependent_children_principal_caregiver_count` (Family)
`wff_person_age` (Family)
`winter_energy_payment_has_dependent_children` (Person)
`winter_energy_payment_person_age` (Person)
`young_person_age_years` (Person)
`young_person_has_dependent_child` (Person)
`young_person_has_dependent_child_for_part6_rates` (Person)
`young_person_is_parent_or_step_parent_of_dependent_child` (Person) | +| `fact:person.age` | `accommodation_supplement_has_dependent_children` (Family)
`accommodation_supplement_has_two_or_more_dependent_children` (Family)
`accommodation_supplement_sole_parent` (Family)
`caregiver_age_years` (Person)
`childcare_assistance_dependent_children_count` (Person)
`community_services_card_dependent_children_count` (Person)
`community_services_card_person_age_years` (Person)
`disability_allowance_dependent_children_count` (Person)
`disability_allowance_person_age_years` (Person)
`disability_allowance_person_is_sole_parent` (Person)
`in_work_tax_credit_allowed_children_count` (Family)
`in_work_tax_credit_person_age` (Family)
`jobseeker_applicant_age` (Person)
`jobseeker_has_dependent_child` (Person)
`jobseeker_support_applicant_age` (Person)
`jobseeker_support_dependent_children_count` (Person)
`nz_super_has_dependent_children` (Person)
`nz_super_person_age` (Person)
`sole_parent_applicant_age` (Person)
`sole_parent_caring_for_dependent_children_under_age_limit` (Person)
`sole_parent_is_mother_or_father_or_child_regarded_as_child` (Person)
`student_allowance_has_children` (Person)
`student_allowance_student_age` (Person)
`supported_living_applicant_age` (Person)
`supported_living_caring_applicant_age` (Person)
`supported_living_caring_applicant_has_dependent_child` (Person)
`supported_living_payment_applicant_age` (Person)
`supported_living_payment_dependent_children_count` (Person)
`wff_dependent_children_principal_caregiver_count` (Family)
`wff_person_age` (Family)
`winter_energy_payment_has_dependent_children` (Person)
`winter_energy_payment_person_age` (Person)
`young_person_age_years` (Person)
`young_person_has_dependent_child` (Person)
`young_person_has_dependent_child_for_part6_rates` (Person)
`young_person_is_parent_or_step_parent_of_dependent_child` (Person) | | `fact:person.legal_marital_status` | `sole_parent_marriage_or_civil_union_dissolved` (Person)
`sole_parent_spouse_or_partner_died` (Person) | -| `fact:person.partner_person_id` | `accommodation_supplement_in_relationship` (Family)
`community_services_card_person_married_or_partnered` (Person)
`disability_allowance_person_in_relationship` (Person)
`has_relationship_partner` (Person)
`jobseeker_support_single` (Person)
`nz_super_in_relationship` (Person)
`nz_super_single` (Person)
`nz_super_single` (Person)
`sole_parent_is_single` (Person)
`student_allowance_has_partner` (Person)
`student_allowance_lives_with_partner` (Person)
`supported_living_payment_single` (Person)
`winter_energy_payment_single` (Person)
`young_person_is_in_relationship` (Person) | -| `fact:person.parent_1_person_id` | `accommodation_supplement_has_dependent_children` (Family)
`childcare_assistance_dependent_children_count` (Person)
`community_services_card_dependent_children_count` (Person)
`disability_allowance_dependent_children_count` (Person)
`disability_allowance_person_is_sole_parent` (Person)
`in_work_tax_credit_allowed_children_count` (Family)
`jobseeker_has_dependent_child` (Person)
`jobseeker_support_dependent_children_count` (Person)
`nz_super_has_dependent_children` (Person)
`sole_parent_caring_for_dependent_children_under_age_limit` (Person)
`sole_parent_is_mother_or_father_or_child_regarded_as_child` (Person)
`student_allowance_has_children` (Person)
`supported_living_caring_applicant_has_dependent_child` (Person)
`supported_living_payment_dependent_children_count` (Person)
`wff_dependent_children_principal_caregiver_count` (Family)
`winter_energy_payment_has_dependent_children` (Person)
`young_person_has_dependent_child` (Person)
`young_person_has_dependent_child_for_part6_rates` (Person)
`young_person_is_parent_or_step_parent_of_dependent_child` (Person) | -| `fact:person.parent_2_person_id` | `accommodation_supplement_has_dependent_children` (Family)
`childcare_assistance_dependent_children_count` (Person)
`community_services_card_dependent_children_count` (Person)
`disability_allowance_dependent_children_count` (Person)
`disability_allowance_person_is_sole_parent` (Person)
`in_work_tax_credit_allowed_children_count` (Family)
`jobseeker_has_dependent_child` (Person)
`jobseeker_support_dependent_children_count` (Person)
`nz_super_has_dependent_children` (Person)
`sole_parent_caring_for_dependent_children_under_age_limit` (Person)
`sole_parent_is_mother_or_father_or_child_regarded_as_child` (Person)
`student_allowance_has_children` (Person)
`supported_living_caring_applicant_has_dependent_child` (Person)
`supported_living_payment_dependent_children_count` (Person)
`wff_dependent_children_principal_caregiver_count` (Family)
`winter_energy_payment_has_dependent_children` (Person)
`young_person_has_dependent_child` (Person)
`young_person_has_dependent_child_for_part6_rates` (Person)
`young_person_is_parent_or_step_parent_of_dependent_child` (Person) | -| `fact:household.reference_person_id` | `accommodation_supplement_weekly_rent_paid` (Family) | -| `fact:person.employment_income` | `acc_earnings_for_earners_levy` (Person)
`acc_employee_earnings_for_low_self_employed_formula` (Person)
`income_tax_employment_income` (Person)
`kiwisaver_gross_salary_or_wages` (Person)
`payroll_donation_person_is_employee` (Person)
`ppl_employees_average_weekly_earnings` (Person)
`student_loan_weekly_earnings` (Person)
`supported_living_blind_average_weekly_earnings` (Person)
`supported_living_blind_beneficiary_employed` (Person)
`supported_living_payment_beneficiary_personal_earnings` (Person) | -| `fact:person.nonfarm_self_employment_income` | `acc_earnings_for_earners_levy` (Person)
`acc_self_employed_earnings_for_earners_levy` (Person)
`family_scheme_activity_income` (Family)
`income_tax_activity_income` (Person) | -| `fact:person.farm_self_employment_income` | `acc_earnings_for_earners_levy` (Person)
`acc_self_employed_earnings_for_earners_levy` (Person)
`family_scheme_activity_income` (Family)
`income_tax_activity_income` (Person) | -| `fact:person.interest_income` | `rwt_gross_interest_and_dividends` (Person) | -| `fact:person.dividend_income` | `rwt_gross_interest_and_dividends` (Person) | +| `fact:person.partner_person_id` | `accommodation_supplement_has_dependent_children` (Family)
`accommodation_supplement_has_two_or_more_dependent_children` (Family)
`accommodation_supplement_in_relationship` (Family)
`accommodation_supplement_sole_parent` (Family)
`community_services_card_person_married_or_partnered` (Person)
`disability_allowance_person_in_relationship` (Person)
`has_relationship_partner` (Person)
`jobseeker_support_single` (Person)
`nz_super_in_relationship` (Person)
`nz_super_single` (Person)
`nz_super_single` (Person)
`sole_parent_is_single` (Person)
`student_allowance_has_partner` (Person)
`student_allowance_lives_with_partner` (Person)
`supported_living_payment_single` (Person)
`winter_energy_payment_single` (Person)
`young_person_is_in_relationship` (Person) | +| `fact:person.parent_1_person_id` | `accommodation_supplement_has_dependent_children` (Family)
`accommodation_supplement_has_two_or_more_dependent_children` (Family)
`accommodation_supplement_sole_parent` (Family)
`childcare_assistance_dependent_children_count` (Person)
`community_services_card_dependent_children_count` (Person)
`disability_allowance_dependent_children_count` (Person)
`disability_allowance_person_is_sole_parent` (Person)
`in_work_tax_credit_allowed_children_count` (Family)
`jobseeker_has_dependent_child` (Person)
`jobseeker_support_dependent_children_count` (Person)
`nz_super_has_dependent_children` (Person)
`sole_parent_caring_for_dependent_children_under_age_limit` (Person)
`sole_parent_is_mother_or_father_or_child_regarded_as_child` (Person)
`student_allowance_has_children` (Person)
`supported_living_caring_applicant_has_dependent_child` (Person)
`supported_living_payment_dependent_children_count` (Person)
`wff_dependent_children_principal_caregiver_count` (Family)
`winter_energy_payment_has_dependent_children` (Person)
`young_person_has_dependent_child` (Person)
`young_person_has_dependent_child_for_part6_rates` (Person)
`young_person_is_parent_or_step_parent_of_dependent_child` (Person) | +| `fact:person.parent_2_person_id` | `accommodation_supplement_has_dependent_children` (Family)
`accommodation_supplement_has_two_or_more_dependent_children` (Family)
`accommodation_supplement_sole_parent` (Family)
`childcare_assistance_dependent_children_count` (Person)
`community_services_card_dependent_children_count` (Person)
`disability_allowance_dependent_children_count` (Person)
`disability_allowance_person_is_sole_parent` (Person)
`in_work_tax_credit_allowed_children_count` (Family)
`jobseeker_has_dependent_child` (Person)
`jobseeker_support_dependent_children_count` (Person)
`nz_super_has_dependent_children` (Person)
`sole_parent_caring_for_dependent_children_under_age_limit` (Person)
`sole_parent_is_mother_or_father_or_child_regarded_as_child` (Person)
`student_allowance_has_children` (Person)
`supported_living_caring_applicant_has_dependent_child` (Person)
`supported_living_payment_dependent_children_count` (Person)
`wff_dependent_children_principal_caregiver_count` (Family)
`winter_energy_payment_has_dependent_children` (Person)
`young_person_has_dependent_child` (Person)
`young_person_has_dependent_child_for_part6_rates` (Person)
`young_person_is_parent_or_step_parent_of_dependent_child` (Person) | +| `fact:household.reference_person_id` | `accommodation_supplement_has_dependent_children` (Family)
`accommodation_supplement_has_two_or_more_dependent_children` (Family)
`accommodation_supplement_owner_weekly_required_payments` (Family)
`accommodation_supplement_sole_parent` (Family)
`accommodation_supplement_weekly_rent_paid` (Family) | +| `fact:person.employment_income` | `acc_earnings_for_earners_levy` (Person)
`acc_employee_earnings_for_low_self_employed_formula` (Person)
`accommodation_supplement_relevant_weekly_income` (Family)
`income_tax_employment_income` (Person)
`jobseeker_support_total_income` (Person)
`kiwisaver_gross_salary_or_wages` (Person)
`main_benefit_income_test_assessable_income` (Person)
`payroll_donation_person_is_employee` (Person)
`ppl_employees_average_weekly_earnings` (Person)
`sole_parent_support_total_income` (Person)
`student_loan_weekly_earnings` (Person)
`supported_living_blind_average_weekly_earnings` (Person)
`supported_living_blind_beneficiary_employed` (Person)
`supported_living_payment_beneficiary_personal_earnings` (Person)
`supported_living_payment_total_income` (Person) | +| `fact:person.nonfarm_self_employment_income` | `acc_earnings_for_earners_levy` (Person)
`acc_self_employed_earnings_for_earners_levy` (Person)
`accommodation_supplement_relevant_weekly_income` (Family)
`family_scheme_activity_income` (Family)
`income_tax_activity_income` (Person)
`jobseeker_support_total_income` (Person)
`main_benefit_income_test_assessable_income` (Person)
`sole_parent_support_total_income` (Person)
`supported_living_payment_total_income` (Person) | +| `fact:person.farm_self_employment_income` | `acc_earnings_for_earners_levy` (Person)
`acc_self_employed_earnings_for_earners_levy` (Person)
`accommodation_supplement_relevant_weekly_income` (Family)
`family_scheme_activity_income` (Family)
`income_tax_activity_income` (Person)
`jobseeker_support_total_income` (Person)
`main_benefit_income_test_assessable_income` (Person)
`sole_parent_support_total_income` (Person)
`supported_living_payment_total_income` (Person) | +| `fact:person.interest_income` | `accommodation_supplement_relevant_weekly_income` (Family)
`jobseeker_support_total_income` (Person)
`main_benefit_income_test_assessable_income` (Person)
`rwt_gross_interest_and_dividends` (Person)
`sole_parent_support_total_income` (Person)
`supported_living_payment_total_income` (Person) | +| `fact:person.dividend_income` | `accommodation_supplement_relevant_weekly_income` (Family)
`jobseeker_support_total_income` (Person)
`main_benefit_income_test_assessable_income` (Person)
`rwt_gross_interest_and_dividends` (Person)
`sole_parent_support_total_income` (Person)
`supported_living_payment_total_income` (Person) | +| `fact:person.rental_income` | `accommodation_supplement_relevant_weekly_income` (Family)
`jobseeker_support_total_income` (Person)
`main_benefit_income_test_assessable_income` (Person)
`sole_parent_support_total_income` (Person)
`supported_living_payment_total_income` (Person) | +| `fact:person.private_pension_income` | `accommodation_supplement_relevant_weekly_income` (Family)
`jobseeker_support_total_income` (Person)
`main_benefit_income_test_assessable_income` (Person)
`sole_parent_support_total_income` (Person)
`supported_living_payment_total_income` (Person) | | `fact:person.public_pension_income` | `independent_earner_tax_credit_receives_new_zealand_superannuation` (Person)
`main_benefit_receiving_new_zealand_superannuation` (Person)
`nz_super_receives_or_becomes_entitled` (Person)
`winter_energy_payment_receiving_new_zealand_superannuation` (Person) | | `fact:person.has_disability` | `disability_allowance_requires_ongoing_support_for_everyday_functions` (Person)
`supported_living_capacity_restricted_by_health_condition_injury_or_disability` (Person) | | `fact:person.education_enrollment` | `community_services_card_student_in_recognised_full_time_tertiary_course` (Person) | | `fact:person.enrolled_full_time` | `community_services_card_student_in_recognised_full_time_tertiary_course` (Person)
`jobseeker_full_time_student` (Person) | | `fact:person.take_up_seed` | `charitable_or_public_benefit_gift_refund_application_made` (Person)
`winter_energy_payment_valid_election_not_to_receive_in_force` (Person) | -| `fact:household.tenure` | `accommodation_supplement_costs_are_homeownership` (Family)
`accommodation_supplement_has_accommodation_costs` (Family)
`accommodation_supplement_rent_paid_to_kainga_ora` (Family)
`accommodation_supplement_rent_paid_to_registered_community_housing_provider_social_housing` (Family) | +| `fact:household.tenure` | `accommodation_supplement_costs_are_homeownership` (Family)
`accommodation_supplement_has_accommodation_costs` (Family)
`accommodation_supplement_owns_premises_and_not_joint_owner_with_resident` (Family)
`accommodation_supplement_rent_paid_to_kainga_ora` (Family)
`accommodation_supplement_rent_paid_to_registered_community_housing_provider_social_housing` (Family) | | `fact:household.rent` | `accommodation_supplement_weekly_rent_paid` (Family) | +| `fact:household.mortgage_interest` | `accommodation_supplement_owner_weekly_required_payments` (Family) | +| `fact:household.mortgage_principal` | `accommodation_supplement_owner_weekly_required_payments` (Family) | ## Concepts no engine input takes | Concept | Reason | |---|---| | `fact:person.sex` | rulespec-nz at 6fe181f has no sex- or gender-dependent input: none of the 668 dense-surface input names or 40 module YAMLs mention sex, gender, male, female, man, woman, wife or husband, and the only related hit, sole_parent_is_mother_or_father_or_child_regarded_as_child (social_security/main_benefits/entitlement.yaml), tests parenthood, which this mapping binds to the parent_of_coresident_child role. | -| `fact:person.rental_income` | rulespec-nz at 6fe181f has no input for rental or other property income: the only activity-income inputs (income_tax_activity_income in income_tax/core/taxable_income.yaml and family_scheme_activity_income in family_scheme_income.yaml) are the MB 3 bridge the module labels 'business and self-employment activity income' and are already bound to self-employment profit; accommodation_supplement_weekly_rent_received_from_other_residents is rent from co-residents of the own dwelling, which the concept excludes; and the benefit income-test inputs are weekly totals of all income. | | `fact:person.realized_capital_gains` | rulespec-nz at 6fe181f has no input for gains on disposals: no rule mentions gains, disposals or the bright-line test, and the only capital input, income_tax_expenditure_or_loss_is_capital_nature (income_tax/core/taxable_income.yaml, s DA 2), classifies expenditure for the general deduction limitation. | -| `fact:person.private_pension_income` | rulespec-nz at 6fe181f has no input the concept determines: the pension inputs are Family-level family-scheme items defined by contribution source or exempt status rather than receipt (family_scheme_superannuation_distribution_income under MB 5 counts only distributions attributable to employer contributions; family_scheme_retirement_scheme_distribution_income is MB 6; family_scheme_exempt_pension_or_annuity_amount is MB 10's exempt pensions and annuities, half included), and community_services_card_raw_superannuation_income is the reg 8(5A) superannuitant income-test total. | | `fact:person.usual_weekly_hours` | rulespec-nz at 6fe181f has no input that takes hours worked: the work-hours inputs are threshold statuses no transform can compute from hours, namely acc_self_employed_more_than_30_hours_per_week (regulations/acc/earners_levy.yaml, reg 6, which also needs self-employment hours alone), jobseeker_in_full_time_employment (main_benefits/entitlement.yaml, s 21), acc_weekly_compensation_is_full_time_earner and the Family-level in_work_tax_credit_normally_full_time_earner; the remaining hours_per_week inputs are childcare, programme-participation or work-capacity hours. | | `fact:person.weeks_worked` | rulespec-nz at 6fe181f has no input the concept determines: the closest, in_work_tax_credit_weekly_periods (Family, income_tax/family_scheme/tax_credits.yaml, s MD 10(2)), counts weeks in which the person or their partner meets the earner requirement, which needs the union of two partners' work weeks that no group rule gives (reference_member drops a working partner; sum_over_members double-counts and adds children's weeks), and minimum_family_tax_credit_weekly_periods and minimum_family_full_time_earner_weeks count only weeks as a full-time earner. | | `fact:person.educational_attainment` | rulespec-nz at 6fe181f has no input for completed education: no rule or input name mentions a qualification, degree, diploma or attainment level; the education inputs are current enrollment or school attendance (this mapping binds some of them to education_enrollment and enrolled_full_time); and young_person_is_study_ready is what youth_young_parent_payment/core.yaml calls an agency-adjudicated composite input (s 50). | -| `fact:household.mortgage_interest` | rulespec-nz at 6fe181f has no input mortgage interest determines on its own: the only owner-cost input, accommodation_supplement_owner_weekly_required_payments (Family, social_security/accommodation_supplement/core.yaml, s 65AAA), is the owner's weekly 'required mortgage-security payments and reasonably required home payments', which needs interest and principal (and other home payments) summed and then scaled by 1/52, and no transform composes a sum with a scale; accommodation_supplement_concessionary_mortgage_payments_to_kainga_ora_or_crown (s 66) needs the lender's identity. | -| `fact:household.mortgage_principal` | rulespec-nz at 6fe181f has no input mortgage principal determines on its own: the only owner-cost input, accommodation_supplement_owner_weekly_required_payments (Family, social_security/accommodation_supplement/core.yaml, s 65AAA), needs principal and interest (and other home payments) summed and then scaled to a weekly amount, and no transform composes a sum with a scale. | | `fact:household.property_tax` | The only rates input, rr_rates_payable_before_rebate (Person) in statutes/rates_rebates/rates_rebate.yaml, cannot be bound at 6fe181f, because the dense engine could not compile that module and nz.json lists it only under cli_walk_inputs, which the adapter's surface loader does not read; once the module compiles, allocate_to_reference_person of property_tax is the candidate (approximate, because the ratepayer need not be the reference person). | ## Inputs no concept covers @@ -152,9 +152,9 @@ Engine version `6fe181fc4c65763150fe97501ef3443774e64869`; concept schema `d4cb0 `acc_standard_earners_levy_including_gst (Person)`, `kiwisaver_employee_deduction (Person)`, `paye_income_tax_on_taxable_portion_of_earnings (Person)`, `student_loan_annual_deduction (Person)` -### nz/statutes/social_security/accommodation_supplement/core.yaml (32) +### nz/statutes/social_security/accommodation_supplement/core.yaml (27) -`accommodation_supplement_additional_resident_in_social_housing (Family)`, `accommodation_supplement_area_let_to_nonresidents (Family)`, `accommodation_supplement_base_rate_weekly_amount (Family)`, `accommodation_supplement_boarder (Family)`, `accommodation_supplement_business_use_area (Family)`, `accommodation_supplement_cash_assets (Family)`, `accommodation_supplement_concessionary_mortgage_payments_to_kainga_ora_or_crown (Family)`, `accommodation_supplement_disability_accommodation_or_care_funded_under_pae_ora_act (Family)`, `accommodation_supplement_has_two_or_more_dependent_children (Family)`, `accommodation_supplement_joint_owner_with_resident (Family)`, `accommodation_supplement_non_beneficiary (Family)`, `accommodation_supplement_non_beneficiary_income_cutout_weekly_amount (Family)`, `accommodation_supplement_owner_weekly_payment_share (Family)`, `accommodation_supplement_owner_weekly_required_payments (Family)`, `accommodation_supplement_owns_premises_and_not_joint_owner_with_resident (Family)`, `accommodation_supplement_partner_receiving_accommodation_supplement (Family)`, `accommodation_supplement_receiving_or_eligible_for_student_allowance_grant (Family)`, `accommodation_supplement_relevant_weekly_income (Family)`, `accommodation_supplement_residential_care_funded_under_residential_care_act (Family)`, `accommodation_supplement_resides_in_area_1 (Family)`, `accommodation_supplement_resides_in_area_2 (Family)`, `accommodation_supplement_resides_in_area_3 (Family)`, `accommodation_supplement_section_68_joint_tenant_exception_applies (Family)`, `accommodation_supplement_self_contained_area_let_to_residents (Family)`, `accommodation_supplement_social_housing_weekly_contributions_paid (Family)`, `accommodation_supplement_sole_parent (Family)`, `accommodation_supplement_super_or_veterans_pension_income_above_schedule_5_limit (Family)`, `accommodation_supplement_total_premises_area (Family)`, `accommodation_supplement_weekly_board_and_lodgings_paid (Family)`, `accommodation_supplement_weekly_boarder_payments_received (Family)`, `accommodation_supplement_weekly_contributions_received_from_additional_residents (Family)`, `accommodation_supplement_weekly_rent_received_from_other_residents (Family)` +`accommodation_supplement_additional_resident_in_social_housing (Family)`, `accommodation_supplement_area_let_to_nonresidents (Family)`, `accommodation_supplement_base_rate_weekly_amount (Family)`, `accommodation_supplement_boarder (Family)`, `accommodation_supplement_business_use_area (Family)`, `accommodation_supplement_cash_assets (Family)`, `accommodation_supplement_concessionary_mortgage_payments_to_kainga_ora_or_crown (Family)`, `accommodation_supplement_disability_accommodation_or_care_funded_under_pae_ora_act (Family)`, `accommodation_supplement_joint_owner_with_resident (Family)`, `accommodation_supplement_non_beneficiary (Family)`, `accommodation_supplement_non_beneficiary_income_cutout_weekly_amount (Family)`, `accommodation_supplement_owner_weekly_payment_share (Family)`, `accommodation_supplement_partner_receiving_accommodation_supplement (Family)`, `accommodation_supplement_receiving_or_eligible_for_student_allowance_grant (Family)`, `accommodation_supplement_residential_care_funded_under_residential_care_act (Family)`, `accommodation_supplement_resides_in_area_1 (Family)`, `accommodation_supplement_resides_in_area_2 (Family)`, `accommodation_supplement_resides_in_area_3 (Family)`, `accommodation_supplement_section_68_joint_tenant_exception_applies (Family)`, `accommodation_supplement_self_contained_area_let_to_residents (Family)`, `accommodation_supplement_social_housing_weekly_contributions_paid (Family)`, `accommodation_supplement_super_or_veterans_pension_income_above_schedule_5_limit (Family)`, `accommodation_supplement_total_premises_area (Family)`, `accommodation_supplement_weekly_board_and_lodgings_paid (Family)`, `accommodation_supplement_weekly_boarder_payments_received (Family)`, `accommodation_supplement_weekly_contributions_received_from_additional_residents (Family)`, `accommodation_supplement_weekly_rent_received_from_other_residents (Family)` ### nz/statutes/social_security/child_disability_allowance/core.yaml (29) @@ -168,9 +168,9 @@ Engine version `6fe181fc4c65763150fe97501ef3443774e64869`; concept schema `d4cb0 `jobseeker_available_for_and_seeking_full_time_employment (Person)`, `jobseeker_has_no_income (Person)`, `jobseeker_health_condition_injury_or_disability_limits_capacity_to_seek_or_undertake_work (Person)`, `jobseeker_in_full_time_employment (Person)`, `jobseeker_income_less_than_zero_rate_cutout (Person)`, `jobseeker_losing_earnings_through_health_condition_or_injury (Person)`, `jobseeker_section_25_discretionary_student_eligibility (Person)`, `jobseeker_taken_reasonable_steps_to_find_full_time_employment (Person)`, `jobseeker_temporary_income_reduces_rate_to_zero_but_otherwise_entitled (Person)`, `jobseeker_unemployed_due_to_strike (Person)`, `jobseeker_unemployed_or_on_leave_for_employment_related_training (Person)`, `jobseeker_willing_and_able_to_undertake_full_time_employment (Person)`, `jobseeker_would_satisfy_available_for_work_but_for_work_test_exemption_circumstances (Person)`, `main_benefit_receiving_another_main_benefit (Person)`, `main_benefit_receiving_veterans_pension (Person)`, `social_security_continuous_residence_years_after_citizen_or_resident (Person)`, `social_security_continuous_residence_years_before_application (Person)`, `social_security_new_zealand_citizen (Person)`, `social_security_only_temporary_entry_class_visa_holder (Person)`, `social_security_ordinarily_resident_in_new_zealand_on_application (Person)`, `social_security_ordinarily_resident_in_reciprocity_country (Person)`, `social_security_refugee_or_protected_person (Person)`, `social_security_regulations_treat_person_as_meeting_residence_requirement (Person)`, `social_security_residence_class_visa_holder (Person)`, `social_security_section_205_override_applies (Person)`, `social_security_unlawfully_resident_or_present_in_new_zealand (Person)`, `sole_parent_living_apart_and_lost_or_lacks_partner_support (Person)`, `sole_parent_lost_regular_support_due_to_partner_sentence_conditions (Person)`, `sole_parent_receiving_sole_parent_support (Person)`, `sole_parent_split_care_court_order_exception_applies (Person)`, `sole_parent_split_care_entitlement_allocated_to_person (Person)`, `sole_parent_split_care_situation_applies (Person)`, `sole_parent_youngest_dependent_child_age (Person)`, `supported_living_cared_person_is_spouse_or_partner (Person)`, `supported_living_cared_person_would_otherwise_receive_institutional_care (Person)`, `supported_living_caring_payment_already_granted (Person)`, `supported_living_caring_required_full_time_care_at_home (Person)`, `supported_living_caring_temporary_continuation_days (Person)`, `supported_living_open_employment_trial_agreed_with_msd (Person)`, `supported_living_open_employment_trial_weeks (Person)`, `supported_living_payment_granted_for_restricted_work_capacity (Person)`, `supported_living_regular_open_employment_capacity_hours_per_week (Person)`, `supported_living_restricted_capacity_or_blindness_self_inflicted_to_qualify (Person)`, `supported_living_restricting_condition_expected_to_continue_required_period (Person)`, `supported_living_terminal_condition (Person)`, `supported_living_totally_blind (Person)` -### nz/statutes/social_security/main_benefits/rates.yaml (19) +### nz/statutes/social_security/main_benefits/rates.yaml (15) -`jobseeker_support_benefit_commenced_on_or_after_1998_07_01 (Person)`, `jobseeker_support_living_with_parent (Person)`, `jobseeker_support_partner_ineligible_due_to_sanction_or_strike (Person)`, `jobseeker_support_partner_receives_main_benefit (Person)`, `jobseeker_support_partner_receives_new_zealand_superannuation_or_veterans_pension (Person)`, `jobseeker_support_total_income (Person)`, `jobseeker_support_transferred_2013_no_dependent_children (Person)`, `jobseeker_support_youngest_dependent_child_age (Person)`, `main_benefit_income_test_assessable_income (Person)`, `sole_parent_support_personal_earnings_used_for_childcare (Person)`, `sole_parent_support_total_income (Person)`, `supported_living_blind_other_weekly_amounts_from_all_sources (Person)`, `supported_living_payment_caring (Person)`, `supported_living_payment_partner_receives_main_benefit (Person)`, `supported_living_payment_partner_receives_new_zealand_superannuation_or_veterans_pension (Person)`, `supported_living_payment_partnered_long_term_residential_care_halving_applies (Person)`, `supported_living_payment_restricted_or_blind (Person)`, `supported_living_payment_total_income (Person)`, `supported_living_payment_totally_blind (Person)` +`jobseeker_support_benefit_commenced_on_or_after_1998_07_01 (Person)`, `jobseeker_support_living_with_parent (Person)`, `jobseeker_support_partner_ineligible_due_to_sanction_or_strike (Person)`, `jobseeker_support_partner_receives_main_benefit (Person)`, `jobseeker_support_partner_receives_new_zealand_superannuation_or_veterans_pension (Person)`, `jobseeker_support_transferred_2013_no_dependent_children (Person)`, `jobseeker_support_youngest_dependent_child_age (Person)`, `sole_parent_support_personal_earnings_used_for_childcare (Person)`, `supported_living_blind_other_weekly_amounts_from_all_sources (Person)`, `supported_living_payment_caring (Person)`, `supported_living_payment_partner_receives_main_benefit (Person)`, `supported_living_payment_partner_receives_new_zealand_superannuation_or_veterans_pension (Person)`, `supported_living_payment_partnered_long_term_residential_care_halving_applies (Person)`, `supported_living_payment_restricted_or_blind (Person)`, `supported_living_payment_totally_blind (Person)` ### nz/statutes/social_security/orphans_unsupported_child_benefit/core.yaml (9) diff --git a/docs/concept-schema-transport-adr.md b/docs/concept-schema-transport-adr.md index 25897ea844..dd93c9858a 100644 --- a/docs/concept-schema-transport-adr.md +++ b/docs/concept-schema-transport-adr.md @@ -2,7 +2,7 @@ **Status:** accepted for the concept schema, the per-engine mappings and their coverage reports, which this change adds. The transport pipeline described -below is the design those pieces serve; none of it is built yet, and no build +below is the design those pieces serve; only its step 6 is built, and no build has migrated. **Date:** 2026-09-28. **Source:** Max's 27 September rulings on international populations and law-anchored concepts. @@ -29,7 +29,9 @@ observes and that keep their meaning across countries. There are 28, on two entities. Engine group entities (tax units, benefit units, SPM units, families) are engine constructs built from relationships. Today the US unit operator builds them from raw CPS roster columns, and the UK adapter requires -them already present. Building them from the concept pointers is future work. +them already present. `microcosm.frame.unit_construction` builds benefit units +from the concept pointers under a declared rule; tax units and SPM units are +not yet built from them. | Topic | Concepts | |---|---| @@ -122,9 +124,9 @@ binds engine inputs to concepts. Each binding states: - the transform: identity, recode, a share pair, a fraction of an amount another input holds whole, a fixed scale (annual to weekly or monthly), a - positivity test, sum, product, allocation to the reference person, a - predicate, a relationship role, a co-resident child count, or a take-up - threshold; + positivity test, sum, a sum then a fixed scale, product, allocation to the + reference person, a predicate, a relationship role, a co-resident child + count, a take-up threshold, or a test of a built unit's composition; - the relation, in the same vocabulary; - the evidence, quoting the engine's own definition or how builds populate the input. @@ -133,10 +135,13 @@ No name is matched. Every concept is either bound or listed as unmapped with a reason, so each gap is a stated decision: policyengine-uk computes the State Pension from a formula-owned variable, for example, and rulespec-nz has no input that reads sex. A binding's `reads` also counts the pointers it uses to -place values, such as the household reference person that allocation reads. Bindings on engine group entities declare -how member values collapse. The mapping only declares that rule. Applying it -needs engine unit membership, which no adapter yet builds from concepts, so -encoding reports these bindings as deferred. +place values, such as the household reference person that allocation reads. +Bindings on engine group entities declare how member values collapse onto the +unit: summed, any member, the head's value, the household's value on every +unit, or a household amount allocated to the unit holding the reference person. +`encode` reports these bindings as deferred, since a concept frame carries no +units; `encode_groups` executes them once a unit-construction step has built +the units (step 6). Inputs differ in what the engine declares about them. PolicyEngine inputs are typed: the engine declares their entity, dtype and period. It has no structured, @@ -206,9 +211,20 @@ it relies on. and tenure and rents. 6. **Encode for the target engine.** The target adapter's mapping turns the calibrated concept frame into its inputs (`ConceptMapping.encode`), for - example `axiom:nz` for rulespec-nz. Bindings on group entities wait for a - unit-construction step that builds the target's units from the concept - pointers; that step does not exist yet. + example `axiom:nz` for rulespec-nz. Bindings on group entities run through + `ConceptMapping.encode_groups` on units built from the concept pointers: + `microcosm.frame.unit_construction` builds benefit units (an adult, their + partner and their dependent children; every other adult is their own unit) + under a rule the country pack declares, since who counts as a dependent + child is law. `encode_groups` also executes declared state bindings, which + feed a group input from model state (a receipt flag the take-up step + assigned, an area the geography step assigned) rather than from a concept, + and takes the scenario knobs (where a household's rent goes, whether an + asset test applies, which area assignment, a rent factor). An Axiom engine + fails a request that reaches an input it was not given, so an Axiom + country pack also closes every root input of each module it binds + (`microcosm.frame.input_closure`): each is encoded or defaulted with its + reason, and the closure is checked against the committed input surface. 7. **Label the tier.** Every release carries a `ContentBasis`: `own_data`, `transport` or `aggregates_only`. The code calls it content basis, not tier, because the repository already has evidence-tier releases, target @@ -244,7 +260,12 @@ These hold for every input and are tested (Hypothesis properties unless noted): from concepts that do not decode cannot be re-encoded), and encoding is deterministic and row-aligned. 4. **Conservation.** Allocation to the reference person preserves every - household amount, and each share pair sums back to its concept. + household amount, and each share pair sums back to its concept. On built + units, allocation to the reference unit puts each household amount on + exactly the unit holding the reference person, its per-adult alternative + spreads it over the household's units, and both preserve it; a sum over + members preserves the members' total, and a household value is constant + within its household. 5. **Monotone take-up.** A higher rate never takes a record out of a program. Rate 0 takes up nobody and rate 1 takes up everybody. Draws lie on [0, 1), are deterministic, do not depend on other records, and differ by program key. @@ -269,7 +290,22 @@ These hold for every input and are tested (Hypothesis properties unless noted): 10. **Serialization.** Every mapping, transform, alignment and coverage report round-trips through JSON. Malformed input raises `ValueError` and never another exception, whatever is corrupted; a missing, unexpected or - wrongly typed field is named. + wrongly typed field is named. So do unit rules, state bindings and input + closures. +11. **Units.** Benefit units partition persons: every person is in exactly + one unit, every unit has a head and at most one partner, partners share a + unit, every dependent child shares a unit with a co-resident parent (or, + with none, with the household reference person), and units nest in + households. Units carry no weights; each takes its household's, so the + sum of unit weights is the sum over households of household weight times + units in the household. Unit ids are deterministic and independent of + row order. Group encoding agrees with a row-by-row reference, and its + unit-composition flags agree with the unit attributes computed + separately. +12. **Input closure** (engine-free). An Axiom country pack's closure puts + every root input of every module it binds, as the committed surface + lists them, in exactly one class (encoded or defaulted), and an input is + concept-encoded exactly when the mapping binds it. ## Consequences @@ -278,13 +314,14 @@ These hold for every input and are tested (Hypothesis properties unless noted): encoding it through the adapter's mapping, one build at a time. - The spec engine's `imputation.yaml` `concepts:` block is unrelated. It groups predictor columns for imputation, and its names are not concept ids. -- Transport still needs four pieces beyond the schema: the donor bank, the - income transformation (step 3), a unit-construction step that builds engine - units from the concept pointers (until it exists, 12 of New Zealand's 79 and - 46 of Belgium's 105 module bindings, and the take-up and housing bindings on - US and UK group entities, stay deferred), and a country pack with geography, - targets and an engine adapter. The schema was the one piece that belongs in - the core from the start. +- Transport still needs three pieces beyond the schema and benefit-unit + construction: the donor bank, the income transformation (step 3), and a + country pack with geography, targets and an engine adapter. New Zealand's + group bindings now execute on benefit units built from the pointers. + Belgium's group bindings, and the take-up and housing bindings on US and UK + group entities, still wait for a step that builds those engines' units from + the pointers, so they stay deferred. The schema was + the one piece that belongs in the core from the start. - Open questions this does not settle: whether take-up propensities persist or are chosen at simulation time (#360), and how the Belgian population-input boundary treats `takes_up_*` names (#824). The Axiom concept-coverage diff --git a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json new file mode 100644 index 0000000000..d5b835c895 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json @@ -0,0 +1,1428 @@ +{ + "format": "microcosm.axiom_input_closure.v1", + "country": "nz", + "content_basis": "transport", + "description": "The input closure for every Axiom module the New Zealand v0 graph binds: each root input of each module's committed input surface is encoded (by a concept binding of the axiom:nz mapping, a state binding declared here, or another graph node), or defaulted with its reason and citation. The axiom adapter omits an input the frame lacks, and the engine fails a request that reaches an absent input, so a run feeds every entry here. The defaults are modelling content listed for the D1 method review.", + "status": "awaiting D1: every default is a proposal (new_default true)", + "work_package": "G4 (unit construction and group encode)", + "mapping": { + "engine": "axiom:nz", + "engine_version": "6fe181fc4c65763150fe97501ef3443774e64869" + }, + "input_surface": { + "fixture": "packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/nz.json", + "rulespec_commit": "6fe181fc4c65763150fe97501ef3443774e64869", + "engine_commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66" + }, + "rulespec_pin": { + "commit": "8dc2507d5f10a0467c285fed54d018ee37246798", + "method_card_row": "MC1", + "note": "The input surface was generated at 6fe181fc, an ancestor of the MC1 pin 8dc2507; git log 6fe181fc..8dc2507 lists no commit touching the four modules, and their SHA-256 values below are the same at both." + }, + "rulespec_paths": { + "nz/statutes/social_security/main_benefits/entitlement.yaml": "94332c778b3b5324798f67cb1b30f59c2af4131be8159455814f453bc814f8d0", + "nz/statutes/social_security/main_benefits/rates.yaml": "7c8ebbc3e62268722529ad790e3f561bbebf807d8a4d67bba24f93cb0809ec39", + "nz/statutes/new_zealand_superannuation/core.yaml": "d8ea0d15d97081031dcba242e66704c800bcf295e19d73bd173c56bd92d497b4", + "nz/statutes/social_security/accommodation_supplement/core.yaml": "f2feca3c2f1de71623ccd577cd2aed20e8822614aac0ccfc767b5faf4eaa83cc" + }, + "receipt_flags": { + "columns": [ + "person.receives_jobseeker_support", + "person.receives_sole_parent_support", + "person.receives_supported_living_payment", + "person.receives_nz_super" + ], + "note": "The person-level receipt flags the receipt layer (takeup.assign@1) writes and the state bindings read. Their names are a contract with that kernel." + }, + "receipt_requirements": { + "note": "What the receipt layer (takeup.assign@1) must enforce because the closure's entitlement defaults leave it to receipt. Each is a modelling default for D1.", + "requirements": [ + "At most one main benefit (Jobseeker Support, Sole Parent Support, Supported Living Payment) per person: Social Security Act 2018 s 18; main_benefit_receiving_another_main_benefit is defaulted false.", + "Never New Zealand Superannuation and a main benefit for the same person: s 18; main_benefit_receiving_new_zealand_superannuation is defaulted false on transported frames.", + "A main benefit only where the rates module's income-tested net weekly payment for it is positive: entitlement treats the s 24 income test and the s 21 work gap as met (jobseeker_income_less_than_zero_rate_cutout true, jobseeker_in_full_time_employment false), and the Sole Parent Support and Supported Living Payment entitlement rules have no income term; the rates module tests the person's own income only." + ] + }, + "engine_optional_evidence": "axiom-rules-engine 04315d94 (the engine that generated the committed input surface) has no optional inputs. A RuleSpec module has no inputs section (RulesDocument holds format, imports, module, units, relations and rules; src/rulespec.rs:361-374), a rule-level default is rejected (RuleSpecError::UnsupportedDefault, src/rulespec.rs:91-94), and the dense batch call collects only the inputs the caller supplies (python-ext/src/lib.rs:458-466). A plain read of an absent input fails every row whose evaluation reaches it with EvalError::MissingInput (src/dense.rs:2790-2803), and that fails the request. The internal input_or_else node that would read a default (src/dense.rs:2805-2811) is never produced from RuleSpec text: formula lowering turns a name no rule defines into a plain input read (src/formula.rs:1400-1413). So no root input of these modules can be verified engine-optional, and this closure classes none as such: an input a run does not encode is defaulted here, with its reason.", + "undetermined_judgments": { + "action": "refuse", + "alternatives": [ + "not_eligible", + "eligible" + ], + "new_default": true, + "reason": "Entitlement judgments come back as codes 1 (holds), -1 (does not hold) and 0 (undetermined). At axiom-rules-engine 04315d94 no dense leaf yields an undetermined value: dense input columns are booleans, integers, floats or text, an absent input is an error rather than an unknown, and and/or/not only pass an undetermined item along. With every root input encoded or defaulted by this closure, a 0 cannot arise, so takeup.assign@1 refuses any 0 it sees instead of guessing: a 0 would mean the engine or the closure changed. The alternatives a review could choose are treating 0 as not entitled (understates receipt) or as entitled (overstates it). microcosm.frame.input_closure.resolve_judgments implements all three." + }, + "knobs": [ + { + "name": "rent_allocation", + "kind": "allocation", + "targets": [ + "accommodation_supplement_weekly_rent_paid" + ], + "central": "reference_unit", + "values": [ + "reference_unit", + "per_adult_share" + ], + "method_card_row": "MC8", + "note": "A household's rent goes to the family containing the reference person (central, a lower bound for other families), or is shared among the household's families in proportion to their adults (scenario S2, the upper sensitivity). Either way the families' rents sum to the household's." + }, + { + "name": "asset_test", + "kind": "switch_zeroes", + "targets": [ + "accommodation_supplement_cash_assets" + ], + "central": true, + "values": [], + "method_card_row": "MC10", + "note": "On (central): each family's summed liquid financial assets meet the reg 15 cash-asset limits. Off (scenario S1, an upper bound): every family's cash assets are set to zero, so the limit never binds." + }, + { + "name": "area_column", + "kind": "state_column", + "targets": [ + "household.as_area" + ], + "central": "as_area", + "values": [ + "as_area", + "as_area_alt" + ], + "method_card_row": "MC9", + "note": "Which household area assignment the area flags read: the central crosswalk (as_area) or the alternative (as_area_alt, scenario S3)." + }, + { + "name": "rent_factor", + "kind": "factor", + "targets": [ + "accommodation_supplement_weekly_rent_paid" + ], + "central": 1.0, + "values": [], + "method_card_row": "MC8", + "note": "A factor on every family's weekly rent: 1.0 centrally; scenario S4's stock adjustment from lodged-bond rents to all rents takes its factor from the B6 evidence." + } + ], + "entries": [ + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_applicant_age", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_available_for_and_seeking_full_time_employment", + "class": "defaulted", + "value": true, + "reason": "Work-test compliance is behavioural and adjudicated by MSD case by case; it is not observed. Treated as met, so entitlement reflects the observed conditions; the take-up draw calibrated to MSD counts (method card MC6) absorbs non-compliance.", + "citation": "Social Security Act 2018 s 22 (jobseeker_available_for_work)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_full_time_student", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_has_dependent_child", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_has_no_income", + "class": "defaulted", + "value": false, + "reason": "Whether the person's income reaches the zero-rate cutout is an output of the rates module, not an input any concept determines; s 24 is carried by jobseeker_income_less_than_zero_rate_cutout instead.", + "citation": "Social Security Act 2018 s 24 (jobseeker_no_or_minimum_income_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_health_condition_injury_or_disability_limits_capacity_to_seek_or_undertake_work", + "class": "defaulted", + "value": false, + "reason": "The main s 22 branch is treated as met, so the health branch is not needed; health-limited capacity enters Supported Living Payment through has_disability instead.", + "citation": "Social Security Act 2018 s 22 (jobseeker_available_for_work)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_in_full_time_employment", + "class": "defaulted", + "value": false, + "reason": "Full-time employment is a threshold status on hours worked that no mapping transform computes (see the mapping's reason for fact:person.usual_weekly_hours). Treated as not in full-time employment, so the s 21 work gap holds for everyone; earnings then screen recipients only through the rates module's income test, which the receipt layer applies (receipt_requirements: a positive net weekly payment).", + "citation": "Social Security Act 2018 s 21 (jobseeker_work_gap)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_income_less_than_zero_rate_cutout", + "class": "defaulted", + "value": true, + "reason": "The s 24 income test is applied by the rates module's income-tested net payment, which is zero for a person whose income reaches the cutout. Entitlement treats s 24 as met and the receipt layer requires a positive net weekly payment (receipt_requirements); without that screen, high earners would count as entitled. The rates module tests the person's own income only (its income inputs are person-level), so a partnered person whose partner earns is screened on their own income: a default for review.", + "citation": "Social Security Act 2018 s 24 (jobseeker_no_or_minimum_income_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_losing_earnings_through_health_condition_or_injury", + "class": "defaulted", + "value": false, + "reason": "Read only with full-time employment, which is defaulted false.", + "citation": "Social Security Act 2018 s 21 (jobseeker_work_gap)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_section_25_discretionary_student_eligibility", + "class": "defaulted", + "value": false, + "reason": "MSD's s 25 discretion for full-time students is not modelled; full-time students (bound from enrolled_full_time) stay ineligible for Jobseeker Support.", + "citation": "Social Security Act 2018 ss 25 and 26 (jobseeker_not_ineligible)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_taken_reasonable_steps_to_find_full_time_employment", + "class": "defaulted", + "value": true, + "reason": "Work-test compliance is behavioural and adjudicated by MSD case by case; it is not observed. Treated as met, so entitlement reflects the observed conditions; the take-up draw calibrated to MSD counts (method card MC6) absorbs non-compliance.", + "citation": "Social Security Act 2018 s 22 (jobseeker_available_for_work)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_temporary_income_reduces_rate_to_zero_but_otherwise_entitled", + "class": "defaulted", + "value": false, + "reason": "s 24 is carried by jobseeker_income_less_than_zero_rate_cutout.", + "citation": "Social Security Act 2018 s 24 (jobseeker_no_or_minimum_income_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_unemployed_due_to_strike", + "class": "defaulted", + "value": false, + "reason": "Strike action is not observed; treated as not applying.", + "citation": "Social Security Act 2018 ss 25 and 26 (jobseeker_not_ineligible)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_unemployed_or_on_leave_for_employment_related_training", + "class": "defaulted", + "value": false, + "reason": "Employment-related training leave is not observed; treated as not applying.", + "citation": "Social Security Act 2018 ss 25 and 26 (jobseeker_not_ineligible)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_willing_and_able_to_undertake_full_time_employment", + "class": "defaulted", + "value": true, + "reason": "Work-test compliance is behavioural and adjudicated by MSD case by case; it is not observed. Treated as met, so entitlement reflects the observed conditions; the take-up draw calibrated to MSD counts (method card MC6) absorbs non-compliance.", + "citation": "Social Security Act 2018 s 22 (jobseeker_available_for_work)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "jobseeker_would_satisfy_available_for_work_but_for_work_test_exemption_circumstances", + "class": "defaulted", + "value": false, + "reason": "Work-test exemptions are adjudicated by MSD and not observed; the main s 22 branch is treated as met, so the exemption branch is not needed.", + "citation": "Social Security Act 2018 s 22 (jobseeker_available_for_work)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "main_benefit_receiving_another_main_benefit", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. Each main benefit's entitlement is assessed on its own; the s 18 one-main-benefit limit is enforced where receipt is assigned (receipt_requirements: at most one main benefit per person).", + "citation": "Social Security Act 2018 s 18 (main_benefit_concurrent_benefit_limit)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "main_benefit_receiving_new_zealand_superannuation", + "class": "defaulted", + "value": false, + "reason": "The mapping binds it to positive public pension income, a donor program receipt that transport drops (concepts.split_for_transport), so on a transported frame that binding never runs. Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. The s 18 bar on a main benefit alongside New Zealand Superannuation is enforced where receipt is assigned (receipt_requirements: never NZS and a main benefit together).", + "citation": "Social Security Act 2018 s 18 (main_benefit_concurrent_benefit_limit)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "main_benefit_receiving_veterans_pension", + "class": "defaulted", + "value": false, + "reason": "Veteran's Pension is not modelled in v0 and the concept schema has no veteran facts.", + "citation": "Social Security Act 2018 s 18 (main_benefit_concurrent_benefit_limit)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_continuous_residence_years_after_citizen_or_resident", + "class": "defaulted", + "value": 2, + "reason": "No citizenship, visa or residence-history data exists in a transported frame (the donor is not New Zealand microdata). Set at the s 16(2) requirement (social_security_residence_years_required is 2 at rulespec-nz 8dc2507), so a citizen meets it; an applicant who arrived within two years is not identified.", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_continuous_residence_years_before_application", + "class": "defaulted", + "value": 0, + "reason": "Read only through the reciprocity-country branch of s 16, which is closed (social_security_ordinarily_resident_in_reciprocity_country is defaulted false).", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_new_zealand_citizen", + "class": "defaulted", + "value": true, + "reason": "No citizenship, visa or residence-history data exists in a transported frame (the donor is not New Zealand microdata). Every person is treated as a New Zealand citizen, so the s 16 residence requirement holds through citizenship (the s 19 lawful-presence requirement holds through the unlawful-presence and temporary-visa defaults); this overstates entitlement for non-citizens and recent migrants.", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_only_temporary_entry_class_visa_holder", + "class": "defaulted", + "value": false, + "reason": "No citizenship, visa or residence-history data exists in a transported frame (the donor is not New Zealand microdata). Nobody is treated as a temporary visa holder.", + "citation": "Social Security Act 2018 s 19 (social_security_lawful_presence_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_ordinarily_resident_in_new_zealand_on_application", + "class": "defaulted", + "value": true, + "reason": "The modelled population is New Zealand's resident population (design weights at the Stats NZ scale), so everyone is ordinarily resident.", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_ordinarily_resident_in_reciprocity_country", + "class": "defaulted", + "value": false, + "reason": "Everyone is ordinarily resident in New Zealand; the reciprocity branch does not apply.", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_refugee_or_protected_person", + "class": "defaulted", + "value": false, + "reason": "No citizenship, visa or residence-history data exists in a transported frame (the donor is not New Zealand microdata). Citizenship carries the s 16 test, so the refugee exception is not needed.", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_regulations_treat_person_as_meeting_residence_requirement", + "class": "defaulted", + "value": false, + "reason": "No regulation-based residence treatment is modelled; citizenship carries the s 16 test.", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_residence_class_visa_holder", + "class": "defaulted", + "value": false, + "reason": "No citizenship, visa or residence-history data exists in a transported frame (the donor is not New Zealand microdata). Citizenship carries the s 16 test (social_security_new_zealand_citizen is defaulted true).", + "citation": "Social Security Act 2018 s 16 (social_security_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_section_205_override_applies", + "class": "defaulted", + "value": false, + "reason": "Read only when the s 19 lawful-presence test fails, which it never does here.", + "citation": "Social Security Act 2018 s 19 (social_security_lawful_presence_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "social_security_unlawfully_resident_or_present_in_new_zealand", + "class": "defaulted", + "value": false, + "reason": "No citizenship, visa or residence-history data exists in a transported frame (the donor is not New Zealand microdata). Nobody is treated as unlawfully present.", + "citation": "Social Security Act 2018 s 19 (social_security_lawful_presence_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_applicant_age", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_caring_for_dependent_children_under_age_limit", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_is_mother_or_father_or_child_regarded_as_child", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_is_single", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_living_apart_and_lost_or_lacks_partner_support", + "class": "defaulted", + "value": false, + "reason": "Single status (bound from the partner pointer) carries the s 30 relationship test; a partner living elsewhere is not observed.", + "citation": "Social Security Act 2018 s 30 (sole_parent_relationship_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_lost_regular_support_due_to_partner_sentence_conditions", + "class": "defaulted", + "value": false, + "reason": "Single status carries the s 30 relationship test; a partner's sentence is not observed.", + "citation": "Social Security Act 2018 s 30 (sole_parent_relationship_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_marriage_or_civil_union_dissolved", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_receiving_sole_parent_support", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. It feeds only the s 33 expiry output for an existing recipient (sole_parent_support_expires_when_youngest_child_turns_age_limit); the child-age limit for a new grant is applied through sole_parent_caring_for_dependent_children_under_age_limit, which the mapping binds.", + "citation": "Social Security Act 2018 s 33 (sole_parent_support_expires_when_youngest_child_turns_age_limit)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_split_care_court_order_exception_applies", + "class": "defaulted", + "value": false, + "reason": "Read only in a split-care situation, which is defaulted false.", + "citation": "Social Security Act 2018 s 32 (sole_parent_split_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_split_care_entitlement_allocated_to_person", + "class": "defaulted", + "value": false, + "reason": "Read only in a split-care situation, which is defaulted false.", + "citation": "Social Security Act 2018 s 32 (sole_parent_split_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_split_care_situation_applies", + "class": "defaulted", + "value": false, + "reason": "The concept schema has no shared-care or custody facts. Treated as no split care, so the s 32 test holds.", + "citation": "Social Security Act 2018 s 32 (sole_parent_split_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_spouse_or_partner_died", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "sole_parent_youngest_dependent_child_age", + "class": "defaulted", + "value": 0, + "reason": "Read only by the s 33 expiry test, which is false while sole_parent_receiving_sole_parent_support is defaulted false.", + "citation": "Social Security Act 2018 s 33 (sole_parent_support_expires_when_youngest_child_turns_age_limit)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_applicant_age", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_capacity_restricted_by_health_condition_injury_or_disability", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_cared_person_is_spouse_or_partner", + "class": "defaulted", + "value": false, + "reason": "The concept schema has no carer or care-recipient facts, so the Supported Living Payment caring branch is closed in v0; this understates SLP carers.", + "citation": "Social Security Act 2018 s 40(1)(a) and (2) (supported_living_caring_full_time_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_cared_person_would_otherwise_receive_institutional_care", + "class": "defaulted", + "value": false, + "reason": "The concept schema has no carer or care-recipient facts, so the Supported Living Payment caring branch is closed in v0; this understates SLP carers.", + "citation": "Social Security Act 2018 s 40(1)(a) and (2) (supported_living_caring_full_time_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_caring_applicant_age", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_caring_applicant_has_dependent_child", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_caring_payment_already_granted", + "class": "defaulted", + "value": false, + "reason": "The concept schema has no carer or care-recipient facts, so the Supported Living Payment caring branch is closed in v0; this understates SLP carers. Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here.", + "citation": "Social Security Act 2018 s 40(1)(a) and (2) (supported_living_caring_full_time_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_caring_required_full_time_care_at_home", + "class": "defaulted", + "value": false, + "reason": "The concept schema has no carer or care-recipient facts, so the Supported Living Payment caring branch is closed in v0; this understates SLP carers.", + "citation": "Social Security Act 2018 s 40(1)(a) and (2) (supported_living_caring_full_time_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_caring_temporary_continuation_days", + "class": "defaulted", + "value": 0, + "reason": "The concept schema has no carer or care-recipient facts, so the Supported Living Payment caring branch is closed in v0; this understates SLP carers. Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here.", + "citation": "Social Security Act 2018 s 40(1)(a) and (2) (supported_living_caring_full_time_care_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_open_employment_trial_agreed_with_msd", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. Nobody is on an s 39 open-employment trial.", + "citation": "Social Security Act 2018 s 39 (supported_living_open_employment_trial_preserves_entitlement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_open_employment_trial_weeks", + "class": "defaulted", + "value": 0, + "reason": "Read only by the s 39 trial output, which is false with no trial agreed.", + "citation": "Social Security Act 2018 s 39 (supported_living_open_employment_trial_preserves_entitlement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_payment_granted_for_restricted_work_capacity", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. It feeds only the s 39 open-employment-trial output.", + "citation": "Social Security Act 2018 s 39 (supported_living_open_employment_trial_preserves_entitlement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_regular_open_employment_capacity_hours_per_week", + "class": "defaulted", + "value": 0, + "reason": "Work capacity is assessed by MSD and not observed (the mapping does not bind work-capacity hours to usual hours worked, see its reason for fact:person.usual_weekly_hours). Set below the s 35(3) severity threshold (15 hours at rulespec-nz 8dc2507), so everyone flagged by has_disability meets the severity test; this overstates SLP entitlement, which the take-up draw calibrated to MSD counts absorbs (method card MC6).", + "citation": "Social Security Act 2018 s 35(3) (supported_living_restricted_work_capacity_severe)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_restricted_capacity_or_blindness_self_inflicted_to_qualify", + "class": "defaulted", + "value": false, + "reason": "Self-inflicted restriction (s 36) is not observed; treated as not applying.", + "citation": "Social Security Act 2018 s 36 (supported_living_restricted_or_blind_not_ineligible)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_restricting_condition_expected_to_continue_required_period", + "class": "defaulted", + "value": true, + "reason": "The concept fact:person.has_disability (bound to supported_living_capacity_restricted_by_health_condition_injury_or_disability) is read as a long-term restriction, so the s 35(2) duration test is treated as met for those it flags; prognosis is not observed.", + "citation": "Social Security Act 2018 s 35(2) (supported_living_restricted_work_capacity_permanent)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_terminal_condition", + "class": "defaulted", + "value": false, + "reason": "Terminal illness is not observed; the s 35(2) duration branch carries the test.", + "citation": "Social Security Act 2018 s 35(2) (supported_living_restricted_work_capacity_permanent)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/entitlement.yaml", + "entity": "Person", + "input": "supported_living_totally_blind", + "class": "defaulted", + "value": false, + "reason": "Total blindness is not a concept; restricted capacity carries s 34(a).", + "citation": "Social Security Act 2018 s 34(a) (supported_living_restricted_or_blind_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_applicant_age", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_benefit_commenced_on_or_after_1998_07_01", + "class": "defaulted", + "value": true, + "reason": "A benefit granted in the modelled tax year commences after 1 July 1998.", + "citation": "Social Security Act 2018 Schedule 4 Part 1 clauses 1 and 3 (jobseeker_support_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_dependent_children_count", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_living_with_parent", + "class": "defaulted", + "value": false, + "reason": "Living with a parent is not read from the parent pointers in v0 (no mapping transform flags a person who has a co-resident parent); this gives an under-20 who lives with a parent the higher rate, overstating their rate.", + "citation": "Social Security Act 2018 Schedule 4 Part 1 clauses 1 and 3 (jobseeker_support_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_partner_ineligible_due_to_sanction_or_strike", + "class": "defaulted", + "value": false, + "reason": "Benefit sanctions and strike action are not observed, so no partner is read as ineligible for either.", + "citation": "Social Security Act 2018 Schedule 4 Part 1 clauses 1 and 3 (jobseeker_support_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_partner_receives_main_benefit", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. A partnered person's rate takes the branch for a partner who receives no main benefit; the receipt layer's net-payment screen reads that rate (receipt_requirements).", + "citation": "Social Security Act 2018 Schedule 4 Part 1 clauses 1 and 3 (jobseeker_support_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_partner_receives_new_zealand_superannuation_or_veterans_pension", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. A partnered person's rate takes the branch for a partner who receives neither; the receipt layer's net-payment screen reads that rate (receipt_requirements).", + "citation": "Social Security Act 2018 Schedule 4 Part 1 clauses 1 and 3 (jobseeker_support_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_single", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_total_income", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_transferred_2013_no_dependent_children", + "class": "defaulted", + "value": false, + "reason": "A 2013 transfer from a former benefit is administrative history, not observed; a new grant is not a transfer.", + "citation": "Social Security Act 2018 Schedule 4 Part 1 clauses 1 and 3 (jobseeker_support_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "jobseeker_support_youngest_dependent_child_age", + "class": "defaulted", + "value": 0, + "reason": "Not read from the built units in v0 (the input is person-level and no mapping transform gives a person their unit's youngest child). Inert at rulespec-nz 8dc2507: the two rates it chooses between, jobseeker_support_single_youngest_child_14_or_older_weekly_rate (clause 1(e)) and jobseeker_support_single_with_dependent_children_weekly_rate (clause 1(f)), are both 521.52, and the income test does not read it.", + "citation": "Social Security Act 2018 Schedule 4 Part 1 clauses 1 and 3 (jobseeker_support_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "main_benefit_income_test_assessable_income", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "sole_parent_support_personal_earnings_used_for_childcare", + "class": "defaulted", + "value": 0, + "reason": "The concept schema has no childcare-cost facts; no childcare earnings disregard is applied.", + "citation": "Social Security Act 2018 Schedule 4 Part 2 clause 2 (sole_parent_support_income_disregarded_for_childcare)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "sole_parent_support_total_income", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_blind_average_weekly_earnings", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_blind_beneficiary_employed", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_blind_other_weekly_amounts_from_all_sources", + "class": "defaulted", + "value": 0, + "reason": "Read only for a totally blind beneficiary, and nobody is flagged totally blind.", + "citation": "Social Security Act 2018 Schedule 4 Part 3 clauses 3 and 4 (supported_living_blind_subsidy_weekly_amount)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_applicant_age", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_beneficiary_personal_earnings", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_caring", + "class": "defaulted", + "value": false, + "reason": "The concept schema has no carer or care-recipient facts, so the Supported Living Payment caring branch is closed in v0; this understates SLP carers.", + "citation": "Social Security Act 2018 Schedule 4 Part 3 clauses 1 and 7 (supported_living_payment_schedule_weekly_rate_before_residential_care_halving)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_dependent_children_count", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_partner_receives_main_benefit", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. A partnered person's Supported Living Payment rate takes the branch for a partner who receives no main benefit; the receipt layer's net-payment screen reads that rate (receipt_requirements).", + "citation": "Social Security Act 2018 Schedule 4 Part 3 clauses 1 and 7 (supported_living_payment_schedule_weekly_rate_before_residential_care_halving)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_partner_receives_new_zealand_superannuation_or_veterans_pension", + "class": "defaulted", + "value": false, + "reason": "Entitlement is assessed before the receipt layer assigns any benefit, so nobody is an existing recipient here. A partnered person's Supported Living Payment rate takes the branch for a partner who receives neither; the receipt layer's net-payment screen reads that rate (receipt_requirements).", + "citation": "Social Security Act 2018 Schedule 4 Part 3 clauses 1 and 7 (supported_living_payment_schedule_weekly_rate_before_residential_care_halving)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_partnered_long_term_residential_care_halving_applies", + "class": "defaulted", + "value": false, + "reason": "The transported population lives in private dwellings (the donor frame is a household survey), so no family is in funded residential or disability care.", + "citation": "Social Security Act 2018 Schedule 4 Part 3 clauses 1 and 5 (supported_living_payment_gross_weekly_rate)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_restricted_or_blind", + "class": "defaulted", + "value": true, + "reason": "With the caring branch closed (supported_living_payment_caring is defaulted false), a Supported Living Payment is the restricted-or-blind payment.", + "citation": "Social Security Act 2018 Schedule 4 Part 3 clause 1 (supported_living_payment_income_disregard_for_restricted_or_blind)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_single", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_total_income", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/main_benefits/rates.yaml", + "entity": "Person", + "input": "supported_living_payment_totally_blind", + "class": "defaulted", + "value": false, + "reason": "Total blindness is not a concept; nobody is read as totally blind, so the blind earnings disregard does not apply.", + "citation": "Social Security Act 2018 Schedule 4 Part 3 clause 1 (supported_living_payment_income_disregard_for_restricted_or_blind)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_all_shared_adults_are_exempt_dependent_children_or_temporary_visitors", + "class": "defaulted", + "value": false, + "reason": "Read only when the residence is shared, which is defaulted false.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 4A (nz_super_living_alone)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_birth_year", + "class": "defaulted", + "value": 1950, + "reason": "Birth year is not a concept (age is, and the mapping binds it to nz_super_person_age). With the residence years set at the longest s 8(4) requirement (below), the birth cohort this selects cannot change the result.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 8(4) (nz_super_standard_total_time_requirement_years)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_born_on_or_before_30_june_of_birth_year", + "class": "defaulted", + "value": true, + "reason": "Selects the cohort with nz_super_birth_year; inert for the same reason.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 8(4) (nz_super_standard_total_time_requirement_years)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_elected_weekly_compensation_instead_of_superannuation", + "class": "defaulted", + "value": false, + "reason": "Electing ACC weekly compensation instead of superannuation (s 7) is not observed.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 7 (nz_super_age_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_ordinarily_resident_in_new_zealand_on_application", + "class": "defaulted", + "value": true, + "reason": "The modelled population is New Zealand's resident population.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 8 (nz_super_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_overseas_application_residence_exception_applies", + "class": "defaulted", + "value": false, + "reason": "Everyone is ordinarily resident in New Zealand, so the s 8 exception for an application made overseas is not needed.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 8 (nz_super_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_person_age", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_principal_residence_is_qualifying_accommodation", + "class": "defaulted", + "value": true, + "reason": "Everyone lives in a private dwelling, which is read as qualifying accommodation for the s 4A living-alone rate.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 4A (nz_super_living_alone)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_resident_present_since_age_20_new_zealand_years", + "class": "defaulted", + "value": 10, + "reason": "Residence history is not observed. Set at the s 8(2)(a) minimum (nz_super_minimum_new_zealand_resident_present_years_since_20 is 10 at rulespec-nz 8dc2507).", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 8 (nz_super_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_resident_present_since_age_20_total_years", + "class": "defaulted", + "value": 20, + "reason": "Residence history is not observed. Set at the longest s 8(4) total-time requirement (nz_super_total_time_requirement_20_years is 20 at rulespec-nz 8dc2507), so everyone of qualifying age meets s 8; this overstates NZS for recent migrants.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 8 (nz_super_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_resident_present_since_age_50_total_years", + "class": "defaulted", + "value": 5, + "reason": "Residence history is not observed. Set at the s 8(3) minimum (nz_super_minimum_resident_present_years_since_50 is 5 at rulespec-nz 8dc2507).", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 8 (nz_super_residential_requirement)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_shares_residence_with_person_at_least_adult_age", + "class": "defaulted", + "value": false, + "reason": "Household composition is not read into this person-level input in v0 (no mapping transform flags another adult in the household), so every single person gets the s 4A living-alone rate; this overstates NZS for singles who share a home. The receipt layer reads NZS eligibility, not the amount; if the reg 17 superannuitant bridge reads the amount (method card MC11), this default matters there too.", + "citation": "New Zealand Superannuation and Retirement Income Act 2001 s 4A (nz_super_living_alone)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/new_zealand_superannuation/core.yaml", + "entity": "Person", + "input": "nz_super_single", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_additional_resident_in_social_housing", + "class": "defaulted", + "value": false, + "reason": "No family is read as an additional resident of a social-housing tenancy: the donor does not record who holds a tenancy. The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 s 65AAA definition of accommodation costs (accommodation_supplement_weekly_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_area_let_to_nonresidents", + "class": "defaulted", + "value": 0, + "reason": "No floor-area facts; with total_premises_area 0 the module takes the branch that applies no area apportionment (Schedule 4 Part 7 clause 7). The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 Schedule 4 Part 7 clause 7 definition of weekly qualifying accommodation costs (accommodation_supplement_weekly_qualifying_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_base_rate_weekly_amount", + "class": "encoded", + "encoded_by": "graph_node", + "node": "nz.as.bridge.base_rate", + "reason": "The reg 17 base rate, computed by the engine-calling bridge node from the main-benefit and family tax credit modules (method card MC11, amendment MC11a; the one sanctioned exception to real-Axiom-only runtime until rulespec-nz encodes reg 17, work package B13). Unit construction and group encoding never compute it." + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_boarder", + "class": "defaulted", + "value": false, + "reason": "Boarders are closed at zero (method card MC8): the donor records no board paid or received between families of one household, so under the central rent allocation a non-reference family's housing costs are zero (under the per-adult-share scenario it takes a share of the rent instead), and the reference family's accommodation income from boarders and other residents is zero; this overstates the reference family's qualifying costs where it takes boarders.", + "citation": "Social Security Act 2018 s 65AAA definition of accommodation costs (accommodation_supplement_weekly_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_business_use_area", + "class": "defaulted", + "value": 0, + "reason": "No floor-area facts; with total_premises_area 0 the module takes the branch that applies no area apportionment (Schedule 4 Part 7 clause 7). The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 Schedule 4 Part 7 clause 7 definition of weekly qualifying accommodation costs (accommodation_supplement_weekly_qualifying_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_cash_assets", + "class": "encoded", + "encoded_by": "concept_binding", + "awaiting": "G3a (transport/liquid-assets-concept): the binding of fact:person.liquid_financial_assets, summed over the family" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_concessionary_mortgage_payments_to_kainga_ora_or_crown", + "class": "defaulted", + "value": false, + "reason": "The lender's identity is not observed, so no mortgage is read as a concessionary Kainga Ora or Crown mortgage (s 66).", + "citation": "Social Security Act 2018 s 66 (accommodation_supplement_social_housing_exclusion_not_applies)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_costs_are_homeownership", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_disability_accommodation_or_care_funded_under_pae_ora_act", + "class": "defaulted", + "value": false, + "reason": "The transported population lives in private dwellings (the donor frame is a household survey), so no family is in funded residential or disability care.", + "citation": "Social Security Act 2018 s 67 (accommodation_supplement_other_funding_exclusion_not_applies)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_has_accommodation_costs", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_has_dependent_children", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_has_two_or_more_dependent_children", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_in_relationship", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_joint_owner_with_resident", + "class": "defaulted", + "value": false, + "reason": "The concept schema has no title or co-ownership facts; an owner-occupier is read as a sole owner. The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 s 65AAA definition of accommodation costs (accommodation_supplement_weekly_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_non_beneficiary", + "class": "encoded", + "encoded_by": "state_binding", + "state_binding": { + "engine_input": "accommodation_supplement_non_beneficiary", + "engine_entity": "Family", + "columns": [ + "person.receives_jobseeker_support", + "person.receives_sole_parent_support", + "person.receives_supported_living_payment", + "person.receives_nz_super" + ], + "test": "any_true", + "group_rule": "any_member", + "negate": true, + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_non_beneficiary", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_non_beneficiary_income_reduction (source: Social Security Regulations 2018 reg 18), which abates the supplement above the non-beneficiary income cutout only when it is true. Read as no member of the family receiving Jobseeker Support, Sole Parent Support, Supported Living Payment or New Zealand Superannuation, from the receipt layer's flags (person.receives_, assigned by take-up as rules-eligible times a draw calibrated to MSD and IRD counts, method card MC6). No receipt is read from the donor. Differences: receipt is modelled, not observed; benefits outside v0 (Youth Payment, Young Parent Payment, Veteran's Pension) are not flags, so their recipients count as non-beneficiaries; and treating a New Zealand Superannuation recipient's family as not a non-beneficiary family is a default for review (method card MC11 assesses superannuitants through the reg 17 superannuitant branch)." + }, + "reason": "Model state, not a concept: see the state binding's note." + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_non_beneficiary_income_cutout_weekly_amount", + "class": "encoded", + "encoded_by": "graph_node", + "node": "nz.as.bridge.cutout", + "reason": "The reg 18 non-beneficiary income cutout, computed by the engine-calling bridge node (method card MC11, amendment MC11a; deleted when B13 encodes reg 18). Unit construction and group encoding never compute it." + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_owner_weekly_payment_share", + "class": "defaulted", + "value": 0, + "reason": "Read only for a joint owner, and nobody is a joint owner. The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 s 65AAA definition of accommodation costs (accommodation_supplement_weekly_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_owner_weekly_required_payments", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_owns_premises_and_not_joint_owner_with_resident", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_partner_receiving_accommodation_supplement", + "class": "defaulted", + "value": false, + "reason": "Partners always share a family under the unit rule (method card MC7) and a family is assessed once, so no partner receives the supplement separately.", + "citation": "Social Security Act 2018 s 67 (accommodation_supplement_other_funding_exclusion_not_applies)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_receiving_or_eligible_for_student_allowance_grant", + "class": "defaulted", + "value": false, + "reason": "Student Allowance is not modelled in v0; a full-time tertiary student who would take the allowance's accommodation benefit instead is counted for the supplement, which overstates it.", + "citation": "Social Security Act 2018 s 67 (accommodation_supplement_other_funding_exclusion_not_applies)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_relevant_weekly_income", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_rent_paid_to_kainga_ora", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_rent_paid_to_registered_community_housing_provider_social_housing", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_residential_care_funded_under_residential_care_act", + "class": "defaulted", + "value": false, + "reason": "The transported population lives in private dwellings (the donor frame is a household survey), so no family is in funded residential or disability care.", + "citation": "Social Security Act 2018 s 67 (accommodation_supplement_other_funding_exclusion_not_applies)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_resides_in_area_1", + "class": "encoded", + "encoded_by": "state_binding", + "state_binding": { + "engine_input": "accommodation_supplement_resides_in_area_1", + "engine_entity": "Family", + "columns": [ + "household.as_area" + ], + "test": "equals", + "value": 1, + "group_rule": "household_value", + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_resides_in_area_1", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_maximum_weekly_rate (source: Social Security Act 2018 Schedule 4 Part 7 clauses 1 to 6), which picks the area's maximum weekly rate; a family in none of areas 1 to 3 takes the area 4 rates. Read from the household's assigned Accommodation Supplement area (household.as_area, 1 to 4) equal to 1. The area is model state: the geography nodes assign it to the household from the territorial-authority-to-area crosswalk with population split shares (method card MC9), not from an address, so every family in a household shares the household's area." + }, + "reason": "Model state, not a concept: see the state binding's note." + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_resides_in_area_2", + "class": "encoded", + "encoded_by": "state_binding", + "state_binding": { + "engine_input": "accommodation_supplement_resides_in_area_2", + "engine_entity": "Family", + "columns": [ + "household.as_area" + ], + "test": "equals", + "value": 2, + "group_rule": "household_value", + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_resides_in_area_2", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_maximum_weekly_rate (source: Social Security Act 2018 Schedule 4 Part 7 clauses 1 to 6), which picks the area's maximum weekly rate; a family in none of areas 1 to 3 takes the area 4 rates. Read from the household's assigned Accommodation Supplement area (household.as_area, 1 to 4) equal to 2. The area is model state: the geography nodes assign it to the household from the territorial-authority-to-area crosswalk with population split shares (method card MC9), not from an address, so every family in a household shares the household's area." + }, + "reason": "Model state, not a concept: see the state binding's note." + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_resides_in_area_3", + "class": "encoded", + "encoded_by": "state_binding", + "state_binding": { + "engine_input": "accommodation_supplement_resides_in_area_3", + "engine_entity": "Family", + "columns": [ + "household.as_area" + ], + "test": "equals", + "value": 3, + "group_rule": "household_value", + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_resides_in_area_3", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_maximum_weekly_rate (source: Social Security Act 2018 Schedule 4 Part 7 clauses 1 to 6), which picks the area's maximum weekly rate; a family in none of areas 1 to 3 takes the area 4 rates. Read from the household's assigned Accommodation Supplement area (household.as_area, 1 to 4) equal to 3. The area is model state: the geography nodes assign it to the household from the territorial-authority-to-area crosswalk with population split shares (method card MC9), not from an address, so every family in a household shares the household's area." + }, + "reason": "Model state, not a concept: see the state binding's note." + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_section_68_joint_tenant_exception_applies", + "class": "defaulted", + "value": false, + "reason": "Read only when a partner receives the supplement separately, which never happens here.", + "citation": "Social Security Act 2018 s 67 (accommodation_supplement_other_funding_exclusion_not_applies)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_self_contained_area_let_to_residents", + "class": "defaulted", + "value": 0, + "reason": "No floor-area facts; with total_premises_area 0 the module takes the branch that applies no area apportionment (Schedule 4 Part 7 clause 7). The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 Schedule 4 Part 7 clause 7 definition of weekly qualifying accommodation costs (accommodation_supplement_weekly_qualifying_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_social_housing_weekly_contributions_paid", + "class": "defaulted", + "value": 0, + "reason": "Read only for an additional resident in social housing, and nobody is. The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 s 65AAA definition of accommodation costs (accommodation_supplement_weekly_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_sole_parent", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_super_or_veterans_pension_income_above_schedule_5_limit", + "class": "defaulted", + "value": false, + "reason": "The superannuitant income test against the Schedule 5 limit is not modelled in v0; the s 67 exclusion is not applied, which overstates the supplement for higher-income superannuitants.", + "citation": "Social Security Act 2018 s 67 (accommodation_supplement_other_funding_exclusion_not_applies)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_total_premises_area", + "class": "defaulted", + "value": 0, + "reason": "No floor-area facts; with total_premises_area 0 the module takes the branch that applies no area apportionment (Schedule 4 Part 7 clause 7). The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 Schedule 4 Part 7 clause 7 definition of weekly qualifying accommodation costs (accommodation_supplement_weekly_qualifying_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_weekly_board_and_lodgings_paid", + "class": "defaulted", + "value": 0, + "reason": "Boarders are closed at zero (method card MC8): the donor records no board paid or received between families of one household, so under the central rent allocation a non-reference family's housing costs are zero (under the per-adult-share scenario it takes a share of the rent instead), and the reference family's accommodation income from boarders and other residents is zero; this overstates the reference family's qualifying costs where it takes boarders.", + "citation": "Social Security Act 2018 s 65AAA definition of accommodation costs (accommodation_supplement_weekly_accommodation_costs)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_weekly_boarder_payments_received", + "class": "defaulted", + "value": 0, + "reason": "Boarders are closed at zero (method card MC8): the donor records no board paid or received between families of one household, so under the central rent allocation a non-reference family's housing costs are zero (under the per-adult-share scenario it takes a share of the rent instead), and the reference family's accommodation income from boarders and other residents is zero; this overstates the reference family's qualifying costs where it takes boarders. The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 Schedule 4 Part 7 clause 7 definition of accommodation income (accommodation_supplement_weekly_accommodation_income)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_weekly_contributions_received_from_additional_residents", + "class": "defaulted", + "value": 0, + "reason": "Boarders are closed at zero (method card MC8): the donor records no board paid or received between families of one household, so under the central rent allocation a non-reference family's housing costs are zero (under the per-adult-share scenario it takes a share of the rent instead), and the reference family's accommodation income from boarders and other residents is zero; this overstates the reference family's qualifying costs where it takes boarders. The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 Schedule 4 Part 7 clause 7 definition of accommodation income (accommodation_supplement_weekly_accommodation_income)", + "new_default": true + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_weekly_rent_paid", + "class": "encoded", + "encoded_by": "concept_binding" + }, + { + "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "entity": "Family", + "input": "accommodation_supplement_weekly_rent_received_from_other_residents", + "class": "defaulted", + "value": 0, + "reason": "Boarders are closed at zero (method card MC8): the donor records no board paid or received between families of one household, so under the central rent allocation a non-reference family's housing costs are zero (under the per-adult-share scenario it takes a share of the rent instead), and the reference family's accommodation income from boarders and other residents is zero; this overstates the reference family's qualifying costs where it takes boarders. The Treasury IncomeExplorer reproduction harness defaults it the same way (TheAxiomFoundation ops nz-lane/emtr_reproduction/run.py, ModelEvaluator.accommodation_values).", + "citation": "Social Security Act 2018 Schedule 4 Part 7 clause 7 definition of accommodation income (accommodation_supplement_weekly_accommodation_income)", + "new_default": true + } + ] +} diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py new file mode 100644 index 0000000000..af1df00d79 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py @@ -0,0 +1,408 @@ +"""The New Zealand input closure, against the committed engine surface. + +``build/nz/axiom_input_closure.json`` decides how every root input of the four +Axiom modules the NZ v0 graph binds is fed: the main-benefit entitlement and +rates modules, New Zealand Superannuation and the Accommodation Supplement. +These tests hold it to the engine-generated input surface and the ``axiom:nz`` +concept mapping, show that encoding plus the closure's defaults supplies every +input a run needs, and replay the Axiom oracle's golden-08 household: the +Accommodation Supplement inputs that unit construction and group encoding +produce from a concept frame equal the inputs the reproduction harness sent +(engine-free), and the real engine turns them into the oracle's weekly amount +(engine-present; skipped without the engine and a rulespec-nz checkout). +""" + +import importlib.util +import json +import os +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from microcosm.frame.adapters.axiom import _batch_from_table, axiom_concept_mapping +from microcosm.frame.adapters.axiom_input_surface import load_axiom_input_surface +from microcosm.frame.concept_mapping import Allocation +from microcosm.frame.concepts import CONCEPTS, ContentBasis, split_for_transport +from microcosm.frame.input_closure import ( + ClosureClass, + EncodedBy, + InputClosure, + UndeterminedAction, + apply_defaults, +) +from microcosm.frame.unit_construction import ( + BenefitUnitRule, + DependentChildRule, + benefit_unit_membership, + build_benefit_units, +) +from test_support.microcosm_frame.concept_frames import concept_frames +from test_support.paths import paths_for + +_BUILD = paths_for("microcosm-build") +_FRAME = paths_for("microcosm-frame") +CLOSURE_PATH = _BUILD.package / "src/microcosm/build/nz/axiom_input_closure.json" +SURFACE_PATH = _FRAME.tests / "fixtures/axiom_input_surfaces/nz.json" +GOLDEN_PATH = ( + _BUILD.tests / "fixtures/nz/axiom_oracle_golden_08_accommodation_supplement.json" +) + +ENTITLEMENT = "nz/statutes/social_security/main_benefits/entitlement.yaml" +RATES = "nz/statutes/social_security/main_benefits/rates.yaml" +NZS = "nz/statutes/new_zealand_superannuation/core.yaml" +AS = "nz/statutes/social_security/accommodation_supplement/core.yaml" +PERSON_MODULES = (ENTITLEMENT, RATES, NZS) + +CLOSURE = InputClosure.from_dict(json.loads(CLOSURE_PATH.read_text(encoding="utf-8"))) +SURFACE = load_axiom_input_surface(SURFACE_PATH) +MAPPING = axiom_concept_mapping("nz") +#: Concept-encoded inputs whose binding lands in a later change (the cash-asset +#: binding arrives with the liquid-asset concept, work package G3a). The +#: closure records the change it waits for; when the binding lands, the +#: closure check reports the entry until its awaiting note is removed. +AWAITING = {entry.ref for entry in CLOSURE.entries if entry.awaiting is not None} +#: Stand-in unit rule for these tests. The NZ rule is spec data (method card +#: MC7, build/nz benefit_unit_rule.json); these properties hold for any rule. +RULE = BenefitUnitRule( + entity="family", + dependent_child=DependentChildRule(max_age=17, financial_independence=None), + unparented_child="reference_person_unit", + split_parents="first_parent_unit", + provenance={"max_age": "test stand-in, not the NZ rule"}, +) +WEEK = 1 / 52 + + +def _payload() -> dict: + return json.loads(CLOSURE_PATH.read_text(encoding="utf-8")) + + +class TestClosure: + def test_the_closure_partitions_every_bound_module(self) -> None: + assert CLOSURE.content_basis is ContentBasis.TRANSPORT + assert CLOSURE.check(MAPPING, SURFACE) == () + + def test_only_the_cash_asset_binding_awaits_a_later_change(self) -> None: + assert {ref.name for ref in AWAITING} <= { + "accommodation_supplement_cash_assets" + } + for entry in CLOSURE.entries: + if entry.awaiting is not None: + assert "G3a" in entry.awaiting + + def test_the_receipt_layer_requirements_are_declared(self) -> None: + requirements = _payload()["receipt_requirements"]["requirements"] + assert len(requirements) == 3 + text = " ".join(requirements) + for name in ( + "main_benefit_receiving_another_main_benefit", + "main_benefit_receiving_new_zealand_superannuation", + "jobseeker_income_less_than_zero_rate_cutout", + ): + assert name in text + (entry,) = ( + entry + for entry in CLOSURE.entries_for(ENTITLEMENT) + if entry.input == name + ) + assert entry.closure_class is ClosureClass.DEFAULTED + assert "receipt_requirements" in entry.reason + + def test_the_four_modules_are_closed_at_their_surface_sizes(self) -> None: + sizes = {path: len(SURFACE.modules[path].refs()) for path in CLOSURE.modules} + assert sizes == {ENTITLEMENT: 60, RATES: 28, NZS: 13, AS: 39} + for path, size in sizes.items(): + assert len(CLOSURE.entries_for(path)) == size + + def test_every_default_is_cited_and_listed_for_review(self) -> None: + for entry in CLOSURE.entries: + if entry.closure_class is ClosureClass.DEFAULTED: + assert entry.new_default, entry.input + assert entry.citation and entry.citation.endswith(")"), entry.input + assert len(entry.reason) > 40, entry.input # a sentence, not a tag + assert len(CLOSURE.new_defaults()) == sum( + entry.closure_class is ClosureClass.DEFAULTED for entry in CLOSURE.entries + ) + + def test_no_input_is_claimed_engine_optional(self) -> None: + # axiom-rules-engine 04315d94 has no optional inputs; the closure says + # why and classes none as engine-optional. + assert not any( + entry.closure_class is ClosureClass.ENGINE_OPTIONAL + for entry in CLOSURE.entries + ) + assert "04315d94" in CLOSURE.engine_optional_evidence + assert CLOSURE.surface_engine_commit.startswith("04315d94") + + def test_undetermined_judgments_are_refused(self) -> None: + assert CLOSURE.undetermined.action is UndeterminedAction.REFUSE + assert CLOSURE.undetermined.new_default + + def test_state_bindings_read_only_receipt_flags_and_the_area(self) -> None: + receipt = set(_payload()["receipt_flags"]["columns"]) + for binding in CLOSURE.state_bindings(): + assert binding.module == AS + assert set(binding.columns) <= receipt | {"household.as_area"} + (non_beneficiary,) = ( + binding + for binding in CLOSURE.state_bindings() + if binding.engine_input == "accommodation_supplement_non_beneficiary" + ) + assert set(non_beneficiary.columns) == receipt + assert non_beneficiary.negate + + def test_the_bridge_values_come_from_their_nodes(self) -> None: + assert { + ref.name: node for ref, node in CLOSURE.graph_node_inputs().items() + } == { + "accommodation_supplement_base_rate_weekly_amount": "nz.as.bridge.base_rate", + "accommodation_supplement_non_beneficiary_income_cutout_weekly_amount": ( + "nz.as.bridge.cutout" + ), + } + + def test_central_knobs_are_the_central_encoding(self) -> None: + assert CLOSURE.group_knobs().targets() == frozenset() + assert dict(CLOSURE.group_knobs().state_columns) == {} + knobs = CLOSURE.group_knobs( + { + "rent_allocation": "per_adult_share", + "asset_test": False, + "area_column": "as_area_alt", + "rent_factor": 1.1, + } + ) + rent = "accommodation_supplement_weekly_rent_paid" + assert dict(knobs.allocation) == {rent: Allocation.PER_ADULT_SHARE} + assert knobs.zeroed == frozenset({"accommodation_supplement_cash_assets"}) + assert dict(knobs.factors) == {rent: 1.1} + assert dict(knobs.state_columns) == { + "household.as_area": "household.as_area_alt" + } + rows = {knob.name: knob.method_card_row for knob in CLOSURE.knobs} + assert rows == { + "rent_allocation": "MC8", + "asset_test": "MC10", + "area_column": "MC9", + "rent_factor": "MC8", + } + + def test_the_closure_resource_is_stable_json(self) -> None: + text = CLOSURE_PATH.read_text(encoding="utf-8") + assert text == json.dumps(json.loads(text), indent=1, ensure_ascii=False) + "\n" + assert InputClosure.from_dict(_payload()).to_dict() == CLOSURE.to_dict() + + +def _with_state(tables, data) -> dict[str, pd.DataFrame]: + person = tables["person"].copy() + household = tables["household"].copy() + receipt = _payload()["receipt_flags"]["columns"] + for column in receipt: + person[column.partition(".")[2]] = np.asarray( + data.draw( + st.lists(st.booleans(), min_size=len(person), max_size=len(person)) + ), + dtype=bool, + ) + for column in ("as_area", "as_area_alt"): + household[column] = np.asarray( + data.draw( + st.lists( + st.integers(1, 4), min_size=len(household), max_size=len(household) + ) + ), + dtype=np.int64, + ) + return {"person": person, "household": household} + + +def _family_inputs(tables, knobs=None) -> pd.DataFrame: + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, RULE) + group = benefit_unit_membership(person, family, membership, RULE) + encoded = MAPPING.encode_groups( + tables, + {"Family": group}, + modules=[AS], + state_bindings=CLOSURE.state_bindings([AS]), + knobs=CLOSURE.group_knobs(knobs), + ) + return apply_defaults(encoded.tables["family"], CLOSURE.defaults(AS, "Family")) + + +class TestExecutableClosure: + @settings(max_examples=60, deadline=None) + @given(tables=concept_frames(max_households=4, max_members=6), data=st.data()) + def test_encoding_and_defaults_supply_every_supplement_input( + self, tables, data + ) -> None: + tables = _with_state(split_for_transport(tables)[0], data) + scenarios = [ + {}, + {"rent_allocation": "per_adult_share"}, + {"area_column": "as_area_alt"}, + {"rent_factor": 1.25}, + ] + if not AWAITING: + scenarios.append({"asset_test": False}) + expected = {name for name in SURFACE.modules[AS].inputs["Family"]} + from_nodes = {ref.name for ref in CLOSURE.graph_node_inputs()} + awaited = {ref.name for ref in AWAITING} + for knobs in scenarios: + family = _family_inputs(tables, knobs) + assert ( + set(family.columns) - {"family_id"} == expected - from_nodes - awaited + ) + for name in expected - from_nodes - awaited: + assert family[name].dtype.kind in "bif", name + + @settings(max_examples=40, deadline=None) + @given(tables=concept_frames(max_households=4, max_members=6)) + def test_encoding_and_defaults_supply_every_person_input(self, tables) -> None: + # The v0 frame is transported: donor program receipts are dropped, so + # a binding that reads one never runs and the closure feeds its input. + transported, dropped = split_for_transport(tables) + assert "public_pension_income" in dropped["person"] + encoded = MAPPING.encode( + transported, + shares={name: 0.5 for name in MAPPING.share_parameters()}, + take_up_rates={name: None for name in MAPPING.take_up_programs()}, + ).tables["person"] + for module in PERSON_MODULES: + names = set(SURFACE.modules[module].inputs["Person"]) + concept_encoded = { + entry.input + for entry in CLOSURE.entries_for(module) + if entry.encoded_by is EncodedBy.CONCEPT_BINDING + } + table = apply_defaults( + encoded.loc[:, sorted(concept_encoded)], + CLOSURE.defaults(module, "Person"), + ) + assert set(table.columns) == names, module + assert all(table[name].dtype.kind in "bif" for name in names), module + + +# --- The Axiom oracle's golden-08 household --------------------------------- + +GOLDEN = json.loads(GOLDEN_PATH.read_text(encoding="utf-8")) + + +def _golden_frame() -> dict[str, pd.DataFrame]: + """Golden-08 as a concept frame: a sole parent and children aged 0, 1, 10. + + The scenario gives no parent age (any adult age gives the same units), a + private rent of 600 a week in area 1, no income of any kind, and a parent + the receipt layer has made a Sole Parent Support recipient. + """ + + ages = [30, 0, 1, 10] + person = pd.DataFrame( + { + "person_id": np.arange(1, 5, dtype=np.int64), + "person_household_id": np.ones(4, dtype=np.int64), + "age": np.asarray(ages, dtype=np.int64), + "partner_person_id": pd.array([None] * 4, dtype="Int64"), + "parent_1_person_id": pd.array([None, 1, 1, 1], dtype="Int64"), + "parent_2_person_id": pd.array([None] * 4, dtype="Int64"), + } + ) + for item in CONCEPTS: + if item.entity == "person" and item.monetary is not None: + person[item.name] = 0.0 + for column in _payload()["receipt_flags"]["columns"]: + person[column.partition(".")[2]] = False + person.loc[0, "receives_sole_parent_support"] = True + household = pd.DataFrame( + { + "household_id": np.ones(1, dtype=np.int64), + "reference_person_id": pd.array([1], dtype="Int64"), + "tenure": ["rented_private"], + "rent": [600.0 * 52], + "mortgage_interest": [0.0], + "mortgage_principal": [0.0], + "property_tax": [0.0], + "as_area": np.asarray([1], dtype=np.int64), + } + ) + return {"person": person, "household": household} + + +def _golden_inputs() -> pd.DataFrame: + family = _family_inputs(_golden_frame()) + for ref, _ in CLOSURE.graph_node_inputs().items(): + # The bridge nodes' outputs: golden-08's engine-computed values. + family[ref.name] = float(GOLDEN["inputs"][ref.name]["value"]) + return family + + +class TestGoldenEight: + def test_group_encoding_reproduces_the_harness_inputs(self) -> None: + family = _golden_inputs() + assert len(family) == 1 + for name, value in GOLDEN["inputs"].items(): + actual = family[name].iloc[0] + if value["kind"] == "bool": + assert family[name].dtype == bool, name + assert bool(actual) is value["value"], name + else: + assert np.isclose( + float(actual), float(value["value"]), rtol=0, atol=1e-9 + ), ( + name, + actual, + value["value"], + ) + # Inputs the payment does not read are still supplied (the closure + # feeds every root input); the harness sent only the 28 it reads. + extra = set(family.columns) - {"family_id"} - set(GOLDEN["inputs"]) + assert extra == { + ref.name + for ref in SURFACE.modules[AS].refs() + if ref.name not in GOLDEN["inputs"] + } - {ref.name for ref in AWAITING} + + +_ENGINE_INSTALLED = importlib.util.find_spec("axiom_rules_engine") is not None +if _ENGINE_INSTALLED: + from axiom_rules_engine.dense import NativeCompiledDenseProgram + + _DENSE_AVAILABLE = NativeCompiledDenseProgram is not None +else: + _DENSE_AVAILABLE = False +RULESPEC_NZ = os.environ.get("POPULACE_RULESPEC_NZ") + + +@pytest.mark.skipif( + not _DENSE_AVAILABLE or not RULESPEC_NZ, + reason=( + "needs axiom_rules_engine with its dense extension and a rulespec-nz " + "checkout at POPULACE_RULESPEC_NZ" + ), +) +def test_the_engine_turns_encoded_inputs_into_the_oracle_amount() -> None: + """Golden-08 through group encoding and the real AS module, within $0.01.""" + + import axiom_rules_engine as engine + + root = Path(RULESPEC_NZ) + program = engine.CompiledDenseProgram.from_file( + root / AS, rulespec_roots=(root,), entity="Family" + ) + assert not program.relations + family = _golden_inputs() + period = GOLDEN["period"] + outputs = program.execute( + period_kind=period["period_kind"], + start=period["start"], + end=period["end"], + inputs=_batch_from_table(family, program.root_inputs), + outputs=list(GOLDEN["outputs"]), + )["outputs"] + for name, expected in GOLDEN["outputs"].items(): + (value,) = np.asarray(outputs[name], dtype=object).ravel() + assert abs(float(value) - float(expected["value"])) <= 0.01, (name, value) diff --git a/packages/microcosm-build/tests/fixtures/nz/axiom_oracle_golden_08_accommodation_supplement.json b/packages/microcosm-build/tests/fixtures/nz/axiom_oracle_golden_08_accommodation_supplement.json new file mode 100644 index 0000000000..12c2b5c240 --- /dev/null +++ b/packages/microcosm-build/tests/fixtures/nz/axiom_oracle_golden_08_accommodation_supplement.json @@ -0,0 +1,180 @@ +{ + "description": "Axiom oracle golden-08: the Accommodation Supplement request the Treasury IncomeExplorer reproduction harness sent for scenario single_parent_three_children_area1_rent at weekly wages 0 (a single parent with children aged 0, 1 and 10, renting at 600 a week in area 1, no income), and the engine's response. Copied verbatim from axiom-oracles; the values are the harness's inputs and the engine's outputs, not Treasury's (Treasury's figure is in golden-outputs.json).", + "entity": "Family", + "entity_id": "single_parent_three_children_area1_rent:as:statutory:0", + "household": { + "area": 1, + "members": [ + { + "age": null, + "role": "sole parent" + }, + { + "age": 0, + "role": "child" + }, + { + "age": 1, + "role": "child" + }, + { + "age": 10, + "role": "child" + } + ], + "note": "From the scenario: Partnered false, Children_ages [0, 1, 10], AS_Accommodation_Costs 600, AS_Accommodation_Rent true, AS_Area 1, gross wages 0. The scenario gives no parent age; the parent receives a main benefit at wages 0 (the request sends non_beneficiary false).", + "receives_main_benefit": true, + "tenure": "rented_private", + "weekly_income": 0, + "weekly_rent": 600 + }, + "inputs": { + "accommodation_supplement_additional_resident_in_social_housing": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_area_let_to_nonresidents": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_base_rate_weekly_amount": { + "kind": "decimal", + "value": "673.8469230769230769230769230769230769231" + }, + "accommodation_supplement_boarder": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_business_use_area": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_costs_are_homeownership": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_has_dependent_children": { + "kind": "bool", + "value": true + }, + "accommodation_supplement_has_two_or_more_dependent_children": { + "kind": "bool", + "value": true + }, + "accommodation_supplement_in_relationship": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_joint_owner_with_resident": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_non_beneficiary": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_non_beneficiary_income_cutout_weekly_amount": { + "kind": "decimal", + "value": "905.0285714285714285714285714285714285714" + }, + "accommodation_supplement_owner_weekly_payment_share": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_owner_weekly_required_payments": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_owns_premises_and_not_joint_owner_with_resident": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_relevant_weekly_income": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_resides_in_area_1": { + "kind": "bool", + "value": true + }, + "accommodation_supplement_resides_in_area_2": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_resides_in_area_3": { + "kind": "bool", + "value": false + }, + "accommodation_supplement_self_contained_area_let_to_residents": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_social_housing_weekly_contributions_paid": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_sole_parent": { + "kind": "bool", + "value": true + }, + "accommodation_supplement_total_premises_area": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_weekly_board_and_lodgings_paid": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_weekly_boarder_payments_received": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_weekly_contributions_received_from_additional_residents": { + "kind": "decimal", + "value": "0" + }, + "accommodation_supplement_weekly_rent_paid": { + "kind": "decimal", + "value": "600" + }, + "accommodation_supplement_weekly_rent_received_from_other_residents": { + "kind": "decimal", + "value": "0" + } + }, + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "outputs": { + "accommodation_supplement_rounded_weekly_payment": { + "kind": "decimal", + "value": "303" + }, + "accommodation_supplement_weekly_amount_before_rounding": { + "kind": "decimal", + "value": "302.07678846153846153846153846" + }, + "accommodation_supplement_weekly_qualifying_accommodation_costs": { + "kind": "decimal", + "value": "600" + } + }, + "period": { + "end": "2027-03-31", + "period_kind": "tax_year", + "start": "2026-04-01" + }, + "source": { + "commit": "6275afcdbe7590d0a56298c4916970a46ed936d0", + "provenance": { + "compiled_sha256": "b1d72c1f4840a1774aefbddc9692e22a79ced26cde6c44efb4c01fc394a15c33", + "engine_binary_sha256": "56fbffea1e0e32c52b6fcbddbca76223bb185b33b49368c288e0c7213b0126e1", + "engine_git_sha": "d59969b53430ae2fd97eb4349d44ad23ce930d85", + "harness": "TheAxiomFoundation/ops/nz-lane/emtr_reproduction/run.py", + "harness_commit": "bcf631b5", + "rulespec_commit": "89a7d25dc03a4d045348620283332de10b1047da" + }, + "repository": "TheAxiomFoundation/axiom-oracles", + "request": "conformance/executable/nz-treasury-incomeexplorer/requests.json requests[8].request", + "response": "conformance/executable/nz-treasury-incomeexplorer/full-responses.json requests[8].response.results[0]", + "scenario": "treasury-emtr-snapshot-expanded.json scenarios[0] in the harness directory (TheAxiomFoundation ops nz-lane/emtr_reproduction)" + } +} diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json index 019cc82342..34b5d1739c 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json @@ -557,20 +557,41 @@ { "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_has_dependent_children", "concepts": [ + "fact:person.age", + "fact:person.partner_person_id", "fact:person.parent_1_person_id", "fact:person.parent_2_person_id", - "fact:person.age" + "fact:household.reference_person_id" ], "engine_entity": "Family", "engine_input": "accommodation_supplement_has_dependent_children", - "group_rule": "any_member", + "group_rule": "reference_member", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", - "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_cash_assets_within_limit (reg 15 asset limits) and accommodation_supplement_maximum_weekly_rate (Schedule 4 Part 7 clauses 1 to 6: 'accommodation_supplement_in_relationship and accommodation_supplement_has_dependent_children' selects the large-family rates). Read as any family member being the parent (birth, adoptive or step) of a co-resident person aged 17 or under. Differences: 'dependent child' is a Social Security Act 2018 Schedule 2 term that this module does not define; the age cut at 17 is this mapping's assumption, not read from the Act; a non-parent caregiver's child is missed and a co-resident child who would not count as dependent is included.", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_cash_assets_within_limit (source: Social Security Act 2018 s 65(1)(b); Social Security Regulations 2018 reg 15), which picks the partnered-or-with-children asset limit, and by accommodation_supplement_maximum_weekly_rate (Schedule 4 Part 7 clauses 1 to 6), where 'accommodation_supplement_in_relationship and accommodation_supplement_has_dependent_children' selects the large-family rates. Read as the family holding at least one dependent child under the country pack's declared benefit-unit rule (microcosm.frame.unit_construction), which builds each family from the relationship pointers, age and the household reference person and states who counts as a dependent child. 'Dependent child' is a Social Security Act 2018 Schedule 2 term that this module does not define and rulespec-nz at 8dc2507 does not encode, so the rule's age limit and placements are spec data, not read from the Act. Differences: the care, maintenance and financial dependence the definition turns on are not observed, so the rule's co-residence test stands in for them; and a child with no co-resident parent is placed in the household reference person's family when the rule says so, which names a caregiver the Act would identify by care.", "relation": "approximate", "transform": { - "kind": "relationship_role", - "max_child_age": 17, - "role": "parent_of_coresident_child" + "feature": "has_dependent_child", + "kind": "unit_composition" + } + }, + { + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_has_two_or_more_dependent_children", + "concepts": [ + "fact:person.age", + "fact:person.partner_person_id", + "fact:person.parent_1_person_id", + "fact:person.parent_2_person_id", + "fact:household.reference_person_id" + ], + "engine_entity": "Family", + "engine_input": "accommodation_supplement_has_two_or_more_dependent_children", + "group_rule": "reference_member", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_maximum_weekly_rate (source: Social Security Act 2018 Schedule 4 Part 7 clauses 1 to 6): 'accommodation_supplement_sole_parent and accommodation_supplement_has_two_or_more_dependent_children' selects the large-family rates, and a sole parent with one dependent child takes the middle-family rates. Read as the family holding at least two dependent children under the country pack's declared benefit-unit rule (microcosm.frame.unit_construction), which builds each family from the relationship pointers, age and the household reference person and states who counts as a dependent child. Differences: as for accommodation_supplement_has_dependent_children, dependency is the rule's co-residence test, not the Act's care and maintenance.", + "relation": "approximate", + "transform": { + "feature": "two_or_more_dependent_children", + "kind": "unit_composition" } }, { @@ -589,6 +610,69 @@ "role": "has_partner" } }, + { + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_owner_weekly_required_payments", + "concepts": [ + "fact:household.mortgage_interest", + "fact:household.mortgage_principal" + ], + "engine_entity": "Family", + "engine_input": "accommodation_supplement_owner_weekly_required_payments", + "group_rule": "allocate_to_reference_unit", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_weekly_accommodation_costs (source: Social Security Act 2018 s 65AAA definition of accommodation costs), whose first branch returns it when accommodation_supplement_owns_premises_and_not_joint_owner_with_resident holds; the result feeds the Schedule 4 Part 7 weekly formula, so the value is weekly although the rule declares period Year. Read as the household's annual mortgage interest plus mortgage principal repaid, summed and scaled by 1/52 (one scaled_sum, since a binding takes one transform), on the family that contains the household reference person and zero on the household's other families. Differences: the module does not define the owner's payments; rates (property tax), insurance and other home payments are not added, which understates owners' costs (method card MC8); the concept is what the household repaid in the year, not what the mortgage requires; and a second family in an owner-occupied household gets zero.", + "relation": "approximate", + "transform": { + "factor": 0.019230769230769232, + "kind": "scaled_sum" + } + }, + { + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_owns_premises_and_not_joint_owner_with_resident", + "concepts": [ + "fact:household.tenure" + ], + "engine_entity": "Family", + "engine_input": "accommodation_supplement_owns_premises_and_not_joint_owner_with_resident", + "group_rule": "household_value", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_weekly_accommodation_costs (source: Social Security Act 2018 s 65AAA definition of accommodation costs): 'if accommodation_supplement_owns_premises_and_not_joint_owner_with_resident: accommodation_supplement_owner_weekly_required_payments'. Read as the household owning its dwelling, outright or with a mortgage. Differences: the concept says nothing about title or co-owners, so a joint owner who lives with another owner is read as a sole owner; and household_value gives true to every family in an owner-occupied household, including a boarder family, whose owner payments are then zero because the household's payments go to the family containing the reference person.", + "relation": "approximate", + "transform": { + "clauses": [ + [ + "fact:household.tenure", + [ + "owned_outright", + "owned_with_mortgage" + ] + ] + ], + "kind": "predicate" + } + }, + { + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_relevant_weekly_income", + "concepts": [ + "fact:person.employment_income", + "fact:person.nonfarm_self_employment_income", + "fact:person.farm_self_employment_income", + "fact:person.interest_income", + "fact:person.dividend_income", + "fact:person.rental_income", + "fact:person.private_pension_income" + ], + "engine_entity": "Family", + "engine_input": "accommodation_supplement_relevant_weekly_income", + "group_rule": "sum_over_members", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_non_beneficiary_income_reduction (source: Social Security Regulations 2018 reg 18): 'if accommodation_supplement_non_beneficiary: max(0, accommodation_supplement_relevant_weekly_income - accommodation_supplement_non_beneficiary_income_cutout_weekly_amount) * accommodation_supplement_non_beneficiary_income_reduction_rate'. Read as the family's annual gross income from employment, non-farm and farm self-employment, interest, dividends, rent and private pensions, summed over its members and scaled by 1/52. Differences: the module does not define relevant income and reg 18's definition is not in it; every member's income is summed, including a dependent child's, and whether reg 18 counts a child's income is not stated here; realized capital gains are left out; public pension income is left out: it is a donor program receipt, which transport drops, and New Zealand Superannuation receipt is the receipt layer's model state, which a non-beneficiary family lacks by construction; and annual flows averaged over 52 weeks stand in for the weekly income at assessment.", + "relation": "approximate", + "transform": { + "factor": 0.019230769230769232, + "kind": "scaled_sum" + } + }, { "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_rent_paid_to_kainga_ora", "concepts": [ @@ -635,6 +719,26 @@ "kind": "predicate" } }, + { + "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_sole_parent", + "concepts": [ + "fact:person.age", + "fact:person.partner_person_id", + "fact:person.parent_1_person_id", + "fact:person.parent_2_person_id", + "fact:household.reference_person_id" + ], + "engine_entity": "Family", + "engine_input": "accommodation_supplement_sole_parent", + "group_rule": "reference_member", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_maximum_weekly_rate (source: Social Security Act 2018 Schedule 4 Part 7 clauses 1 to 6), which gives a sole parent with two or more dependent children the large-family rates and one with one dependent child the middle-family rates. Read as the family having one adult (no partner in it) and at least one dependent child under the country pack's declared benefit-unit rule (microcosm.frame.unit_construction), which builds each family from the relationship pointers, age and the household reference person and states who counts as a dependent child. Differences: the module does not define a sole parent; the concept pointers are co-residence based, so a parent whose partner lives elsewhere counts here, and so does a family head with no partner who cares for a child the rule places with them (for example a household reference person and a younger relative with no co-resident parent).", + "relation": "approximate", + "transform": { + "feature": "sole_parent", + "kind": "unit_composition" + } + }, { "canonical_input": "nz:statutes/social_security/accommodation_supplement/core#input.accommodation_supplement_weekly_rent_paid", "concepts": [ @@ -783,7 +887,7 @@ "engine_entity": "Person", "engine_input": "main_benefit_receiving_new_zealand_superannuation", "module": "nz/statutes/social_security/main_benefits/entitlement.yaml", - "note": "Consumed in nz/statutes/social_security/main_benefits/entitlement.yaml by main_benefit_concurrent_benefit_limit (source: Social Security Act 2018 s 18), whose formula includes 'and not main_benefit_receiving_new_zealand_superannuation'. Read as positive(public_pension_income): the concept's definition names NZ Superannuation among the social-insurance or state old-age pensions it holds. Differences: the concept also holds any other such pension (an overseas state pension, or a Veteran's Pension if the source records it as a public pension), so the flag can be true for someone who receives no NZ Superannuation. s 18 bars receiving a main benefit and NZ Superannuation at the same time, while the concept counts receipt at any time in the reference year. The concept's transport rule is drop, so on a transported frame encode skips this binding and the input keeps its default; it executes only on frames built from NZ's own data.", + "note": "Consumed in nz/statutes/social_security/main_benefits/entitlement.yaml by main_benefit_concurrent_benefit_limit (source: Social Security Act 2018 s 18), whose formula includes 'and not main_benefit_receiving_new_zealand_superannuation'. Read as positive(public_pension_income): the concept's definition names NZ Superannuation among the social-insurance or state old-age pensions it holds. Differences: the concept also holds any other such pension (an overseas state pension, or a Veteran's Pension if the source records it as a public pension), so the flag can be true for someone who receives no NZ Superannuation. s 18 bars receiving a main benefit and NZ Superannuation at the same time, while the concept counts receipt at any time in the reference year. The concept's transport rule is drop, so on a transported frame encode skips this binding; the engine has no default for an absent input, so a transport run's input closure (build/nz/axiom_input_closure.json) feeds it instead. The binding executes only on frames built from NZ's own data.", "relation": "approximate", "transform": { "kind": "positive" @@ -1003,6 +1107,69 @@ "role": "no_partner" } }, + { + "canonical_input": "nz:statutes/social_security/main_benefits/rates#input.jobseeker_support_total_income", + "concepts": [ + "fact:person.employment_income", + "fact:person.nonfarm_self_employment_income", + "fact:person.farm_self_employment_income", + "fact:person.interest_income", + "fact:person.dividend_income", + "fact:person.rental_income", + "fact:person.private_pension_income" + ], + "engine_entity": "Person", + "engine_input": "jobseeker_support_total_income", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "note": "Consumed in nz/statutes/social_security/main_benefits/rates.yaml by jobseeker_support_weekly_income_test_reduction (source: Social Security Act 2018 Schedule 2 Income Tests 1, 3 and 4; Schedule 4 Part 1), which abates the Jobseeker Support rate above main_benefit_income_test_lower_weekly_threshold. Read as the person's own annual gross income from employment, non-farm and farm self-employment, interest, dividends, rent and private pensions, summed and scaled by 1/52. Differences: the module does not define the income it tests; the input is a person-level value, so a partner's income is not added, and whether the test reads a couple's joint income is not stated in this module; main-benefit and New Zealand Superannuation amounts are model state, not concepts, so neither is in the sum, and public pension income, a donor program receipt that transport drops, is left out; realized capital gains are left out; and annual flows averaged over 52 weeks stand in for the weekly income at assessment.", + "relation": "approximate", + "transform": { + "factor": 0.019230769230769232, + "kind": "scaled_sum" + } + }, + { + "canonical_input": "nz:statutes/social_security/main_benefits/rates#input.main_benefit_income_test_assessable_income", + "concepts": [ + "fact:person.employment_income", + "fact:person.nonfarm_self_employment_income", + "fact:person.farm_self_employment_income", + "fact:person.interest_income", + "fact:person.dividend_income", + "fact:person.rental_income", + "fact:person.private_pension_income" + ], + "engine_entity": "Person", + "engine_input": "main_benefit_income_test_assessable_income", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "note": "Consumed in nz/statutes/social_security/main_benefits/rates.yaml by main_benefit_income_test_1_weekly_reduction to main_benefit_income_test_4_weekly_reduction (source: Social Security Act 2018 Schedule 2 definitions of Income Tests 1 to 4), which abate above main_benefit_income_test_lower_weekly_threshold. Read as the person's own annual gross income from employment, non-farm and farm self-employment, interest, dividends, rent and private pensions, summed and scaled by 1/52. Differences: the module does not define the income it tests; the input is a person-level value, so a partner's income is not added, and whether the test reads a couple's joint income is not stated in this module; main-benefit and New Zealand Superannuation amounts are model state, not concepts, so neither is in the sum, and public pension income, a donor program receipt that transport drops, is left out; realized capital gains are left out; and annual flows averaged over 52 weeks stand in for the weekly income at assessment.", + "relation": "approximate", + "transform": { + "factor": 0.019230769230769232, + "kind": "scaled_sum" + } + }, + { + "canonical_input": "nz:statutes/social_security/main_benefits/rates#input.sole_parent_support_total_income", + "concepts": [ + "fact:person.employment_income", + "fact:person.nonfarm_self_employment_income", + "fact:person.farm_self_employment_income", + "fact:person.interest_income", + "fact:person.dividend_income", + "fact:person.rental_income", + "fact:person.private_pension_income" + ], + "engine_entity": "Person", + "engine_input": "sole_parent_support_total_income", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "note": "Consumed in nz/statutes/social_security/main_benefits/rates.yaml by sole_parent_support_income_for_income_test (source: Social Security Act 2018 Schedule 4 Part 2 clauses 1 and 2), 'max(0, sole_parent_support_total_income - sole_parent_support_income_disregarded_for_childcare)'. Read as the person's own annual gross income from employment, non-farm and farm self-employment, interest, dividends, rent and private pensions, summed and scaled by 1/52. Differences: the module does not define the income it tests; the input is a person-level value, so a partner's income is not added, and whether the test reads a couple's joint income is not stated in this module; main-benefit and New Zealand Superannuation amounts are model state, not concepts, so neither is in the sum, and public pension income, a donor program receipt that transport drops, is left out; realized capital gains are left out; and annual flows averaged over 52 weeks stand in for the weekly income at assessment.", + "relation": "approximate", + "transform": { + "factor": 0.019230769230769232, + "kind": "scaled_sum" + } + }, { "canonical_input": "nz:statutes/social_security/main_benefits/rates#input.supported_living_blind_average_weekly_earnings", "concepts": [ @@ -1093,6 +1260,27 @@ "role": "no_partner" } }, + { + "canonical_input": "nz:statutes/social_security/main_benefits/rates#input.supported_living_payment_total_income", + "concepts": [ + "fact:person.employment_income", + "fact:person.nonfarm_self_employment_income", + "fact:person.farm_self_employment_income", + "fact:person.interest_income", + "fact:person.dividend_income", + "fact:person.rental_income", + "fact:person.private_pension_income" + ], + "engine_entity": "Person", + "engine_input": "supported_living_payment_total_income", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "note": "Consumed in nz/statutes/social_security/main_benefits/rates.yaml by supported_living_payment_income_for_income_test (source: Social Security Act 2018 Schedule 4 Part 3 clause 1), 'max(0, supported_living_payment_total_income - supported_living_payment_income_disregard_for_restricted_or_blind)'. Read as the person's own annual gross income from employment, non-farm and farm self-employment, interest, dividends, rent and private pensions, summed and scaled by 1/52. Differences: the module does not define the income it tests; the input is a person-level value, so a partner's income is not added, and whether the test reads a couple's joint income is not stated in this module; main-benefit and New Zealand Superannuation amounts are model state, not concepts, so neither is in the sum, and public pension income, a donor program receipt that transport drops, is left out; realized capital gains are left out; and annual flows averaged over 52 weeks stand in for the weekly income at assessment.", + "relation": "approximate", + "transform": { + "factor": 0.019230769230769232, + "kind": "scaled_sum" + } + }, { "canonical_input": "nz:statutes/social_security/orphans_unsupported_child_benefit/core#input.caregiver_age_years", "concepts": [ @@ -1292,13 +1480,9 @@ "input_declaration": "usage_inferred", "structural_inputs": [], "unmapped": { - "fact:household.mortgage_interest": "rulespec-nz at 6fe181f has no input mortgage interest determines on its own: the only owner-cost input, accommodation_supplement_owner_weekly_required_payments (Family, social_security/accommodation_supplement/core.yaml, s 65AAA), is the owner's weekly 'required mortgage-security payments and reasonably required home payments', which needs interest and principal (and other home payments) summed and then scaled by 1/52, and no transform composes a sum with a scale; accommodation_supplement_concessionary_mortgage_payments_to_kainga_ora_or_crown (s 66) needs the lender's identity.", - "fact:household.mortgage_principal": "rulespec-nz at 6fe181f has no input mortgage principal determines on its own: the only owner-cost input, accommodation_supplement_owner_weekly_required_payments (Family, social_security/accommodation_supplement/core.yaml, s 65AAA), needs principal and interest (and other home payments) summed and then scaled to a weekly amount, and no transform composes a sum with a scale.", "fact:household.property_tax": "The only rates input, rr_rates_payable_before_rebate (Person) in statutes/rates_rebates/rates_rebate.yaml, cannot be bound at 6fe181f, because the dense engine could not compile that module and nz.json lists it only under cli_walk_inputs, which the adapter's surface loader does not read; once the module compiles, allocate_to_reference_person of property_tax is the candidate (approximate, because the ratepayer need not be the reference person).", "fact:person.educational_attainment": "rulespec-nz at 6fe181f has no input for completed education: no rule or input name mentions a qualification, degree, diploma or attainment level; the education inputs are current enrollment or school attendance (this mapping binds some of them to education_enrollment and enrolled_full_time); and young_person_is_study_ready is what youth_young_parent_payment/core.yaml calls an agency-adjudicated composite input (s 50).", - "fact:person.private_pension_income": "rulespec-nz at 6fe181f has no input the concept determines: the pension inputs are Family-level family-scheme items defined by contribution source or exempt status rather than receipt (family_scheme_superannuation_distribution_income under MB 5 counts only distributions attributable to employer contributions; family_scheme_retirement_scheme_distribution_income is MB 6; family_scheme_exempt_pension_or_annuity_amount is MB 10's exempt pensions and annuities, half included), and community_services_card_raw_superannuation_income is the reg 8(5A) superannuitant income-test total.", "fact:person.realized_capital_gains": "rulespec-nz at 6fe181f has no input for gains on disposals: no rule mentions gains, disposals or the bright-line test, and the only capital input, income_tax_expenditure_or_loss_is_capital_nature (income_tax/core/taxable_income.yaml, s DA 2), classifies expenditure for the general deduction limitation.", - "fact:person.rental_income": "rulespec-nz at 6fe181f has no input for rental or other property income: the only activity-income inputs (income_tax_activity_income in income_tax/core/taxable_income.yaml and family_scheme_activity_income in family_scheme_income.yaml) are the MB 3 bridge the module labels 'business and self-employment activity income' and are already bound to self-employment profit; accommodation_supplement_weekly_rent_received_from_other_residents is rent from co-residents of the own dwelling, which the concept excludes; and the benefit income-test inputs are weekly totals of all income.", "fact:person.sex": "rulespec-nz at 6fe181f has no sex- or gender-dependent input: none of the 668 dense-surface input names or 40 module YAMLs mention sex, gender, male, female, man, woman, wife or husband, and the only related hit, sole_parent_is_mother_or_father_or_child_regarded_as_child (social_security/main_benefits/entitlement.yaml), tests parenthood, which this mapping binds to the parent_of_coresident_child role.", "fact:person.usual_weekly_hours": "rulespec-nz at 6fe181f has no input that takes hours worked: the work-hours inputs are threshold statuses no transform can compute from hours, namely acc_self_employed_more_than_30_hours_per_week (regulations/acc/earners_levy.yaml, reg 6, which also needs self-employment hours alone), jobseeker_in_full_time_employment (main_benefits/entitlement.yaml, s 21), acc_weekly_compensation_is_full_time_earner and the Family-level in_work_tax_credit_normally_full_time_earner; the remaining hours_per_week inputs are childcare, programme-participation or work-capacity hours.", "fact:person.weeks_worked": "rulespec-nz at 6fe181f has no input the concept determines: the closest, in_work_tax_credit_weekly_periods (Family, income_tax/family_scheme/tax_credits.yaml, s MD 10(2)), counts weeks in which the person or their partner meets the earner requirement, which needs the union of two partners' work weeks that no group rule gives (reference_member drops a working partner; sum_over_members double-counts and adds children's weeks), and minimum_family_tax_credit_weekly_periods and minimum_family_full_time_earner_weeks count only weeks as a full-time earner." diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 97d0d2f9b5..0b594c86f7 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -12,10 +12,13 @@ columns, and :meth:`ConceptMapping.decode` recovers every concept whose bindings are invertible. Bindings on engine group entities (tax units, benefit units, SPM units, families) need unit membership that a concept frame -does not carry; they declare a :class:`GroupRule` and are reported as -deferred rather than executed. No adapter applies group rules yet: that is -the work of a future unit-construction step that builds engine units from -the relationship pointers. +does not carry; they declare a :class:`GroupRule`, and :meth:`encode` +reports them as deferred. :meth:`ConceptMapping.encode_groups` executes them +once a unit-construction step has built the units from the relationship +pointers (:mod:`microcosm.frame.unit_construction` builds benefit units) and +hands their :class:`GroupMembership` over. It also executes declared state +bindings (:class:`StateBinding`), which feed group inputs from model state +(receipt flags, an assigned area) rather than from concepts. :func:`coverage_report` compares a mapping with an engine's input surface: which inputs each concept feeds, which inputs no concept covers, and which @@ -58,12 +61,16 @@ __all__ = [ "AllocateToReferencePerson", + "Allocation", "ConceptMappedEngine", "ConceptMapping", "CoresidentChildCount", "CoverageReport", + "EncodedGroupInputs", "EncodedInputs", "Fraction", + "GroupKnobs", + "GroupMembership", "GroupRule", "Identity", "InputBinding", @@ -77,9 +84,15 @@ "Recode", "RelationshipRole", "Role", + "ScaledSum", "Share", + "StateBinding", + "StateTest", "Sum", "TakeUpThreshold", + "UnitComposition", + "UnitFeature", + "UnitRole", "bind", "coverage_report", "rules_engine_input_refs", @@ -110,8 +123,8 @@ class InputDeclaration(StrEnum): class GroupRule(StrEnum): """How a binding on an engine group entity collapses member values. - This module declares the rule; it does not apply it. Applying it needs - engine unit membership, which no adapter builds from concepts yet. + :meth:`ConceptMapping.encode_groups` applies every rule except + :attr:`PERSON_ROLE`, given the units' :class:`GroupMembership`. """ #: Sum the members' values. @@ -204,6 +217,25 @@ class Sum: """The engine input is the sum of several amount concepts.""" +@dataclass(frozen=True) +class ScaledSum: + """The engine input is the sum of several amount concepts times a factor. + + A rule can state a total in another period than the concepts: the weekly + sum of two annual home payments is ``(interest + principal) / 52``. One + binding has one transform, so :class:`Sum` and :class:`Scale` cannot be + chained; this is the two applied in that order. Not invertible. + """ + + factor: float + + def __post_init__(self) -> None: + factor = float(self.factor) + if not math.isfinite(factor) or factor <= 0: + raise ValueError(f"A scale factor must be finite and positive: {factor}.") + object.__setattr__(self, "factor", factor) + + @dataclass(frozen=True) class Product: """The engine input is the product of two concepts (hours x weeks).""" @@ -324,6 +356,54 @@ class TakeUpThreshold: program: str +class UnitRole(StrEnum): + """A person's place in an engine unit built from the pointers.""" + + #: The unit's reference member. + HEAD = "head" + #: The head's co-resident partner, in the head's unit. + PARTNER = "partner" + #: A dependent child of the unit's adults (or placed with them). + DEPENDENT_CHILD = "dependent_child" + + +class UnitFeature(StrEnum): + """A composition test on a built unit, read from its members' roles.""" + + #: The unit has a partner besides its head. + HAS_PARTNER = "has_partner" + #: The unit has at least one dependent child. + HAS_DEPENDENT_CHILD = "has_dependent_child" + #: The unit has at least two dependent children. + TWO_OR_MORE_DEPENDENT_CHILDREN = "two_or_more_dependent_children" + #: The unit has one adult (no partner) and at least one dependent child. + SOLE_PARENT = "sole_parent" + + +@dataclass(frozen=True) +class UnitComposition: + """A boolean group input: the built unit passes a composition test. + + The unit is built from the relationship pointers, age and the household + reference person under a declared rule (who counts as a dependent child + is law, so it lives in the rule, not here; a rule with a + financial-independence test also reads usual weekly hours). The value is + the unit's own, the same for every member, so its binding takes the + :attr:`GroupRule.REFERENCE_MEMBER` rule. It needs unit membership and runs + only in :meth:`ConceptMapping.encode_groups`. + """ + + feature: UnitFeature + + def __post_init__(self) -> None: + object.__setattr__(self, "feature", UnitFeature(self.feature)) + + @property + def concepts(self) -> tuple[str, ...]: + """The concepts unit construction reads.""" + return _UNIT_CONCEPTS + + Transform = ( Identity | Recode @@ -333,11 +413,13 @@ class TakeUpThreshold: | Scale | Sum | Product + | ScaledSum | AllocateToReferencePerson | Predicate | RelationshipRole | CoresidentChildCount | TakeUpThreshold + | UnitComposition ) #: Serialized transform kind -> transform class. @@ -355,6 +437,8 @@ class TakeUpThreshold: "fraction": Fraction, "positive": Positive, "scale": Scale, + "scaled_sum": ScaledSum, + "unit_composition": UnitComposition, } _KIND_BY_TRANSFORM: dict[type, str] = { cls: kind for kind, cls in _TRANSFORM_KINDS.items() @@ -383,8 +467,10 @@ def transform_to_dict(transform: Transform) -> dict[str, object]: out["max_child_age"] = transform.max_child_age elif isinstance(transform, Fraction): out["parameter"] = transform.parameter - elif isinstance(transform, Scale): + elif isinstance(transform, Scale | ScaledSum): out["factor"] = transform.factor + elif isinstance(transform, UnitComposition): + out["feature"] = transform.feature.value elif isinstance(transform, CoresidentChildCount) and transform.max_age is not None: out["max_age"] = transform.max_age elif isinstance(transform, TakeUpThreshold): @@ -399,6 +485,8 @@ def transform_to_dict(transform: Transform) -> dict[str, object]: "relationship_role": frozenset({"role", "max_child_age"}), "fraction": frozenset({"parameter"}), "scale": frozenset({"factor"}), + "scaled_sum": frozenset({"factor"}), + "unit_composition": frozenset({"feature"}), "coresident_child_count": frozenset({"max_age"}), "take_up_threshold": frozenset({"program"}), } @@ -480,11 +568,13 @@ def optional_age(name: str) -> int | None: ) if kind == "fraction": return Fraction(parameter=text("parameter")) - if kind == "scale": + if kind in ("scale", "scaled_sum"): factor = required("factor") if isinstance(factor, bool) or not isinstance(factor, int | float): raise ValueError("A scale factor must be a number.") - return Scale(factor=factor) + return _TRANSFORM_KINDS[kind](factor=factor) + if kind == "unit_composition": + return UnitComposition(feature=UnitFeature(text("feature"))) if kind == "coresident_child_count": return CoresidentChildCount(max_age=optional_age("max_age")) if kind == "take_up_threshold": @@ -511,6 +601,14 @@ def optional_age(name: str) -> int | None: Role.NO_PARTNER: ("fact:person.partner_person_id",), } _PARENT_CONCEPTS = ("fact:person.parent_1_person_id", "fact:person.parent_2_person_id") +#: What unit construction reads: the person pointers, age and the household +#: reference person (:mod:`microcosm.frame.unit_construction`). +_UNIT_CONCEPTS = ( + "fact:person.age", + "fact:person.partner_person_id", + *_PARENT_CONCEPTS, + "fact:household.reference_person_id", +) class InputRef(NamedTuple): @@ -707,7 +805,9 @@ def concept_entity(self) -> str: """The entity whose rows the transform produces values on.""" if isinstance(self.transform, AllocateToReferencePerson): return "person" - if isinstance(self.transform, RelationshipRole | CoresidentChildCount): + if isinstance( + self.transform, RelationshipRole | CoresidentChildCount | UnitComposition + ): return "person" entities = {concept(concept_id).entity for concept_id in self.concepts} if len(entities) != 1: @@ -771,7 +871,9 @@ def _check_transform_arity(binding: InputBinding) -> None: f"Binding for {name!r}: a recode must map exactly the " f"concept's domain {list(domain)}." ) - if isinstance(transform, Share | Fraction | Sum | AllocateToReferencePerson): + if isinstance( + transform, Share | Fraction | Sum | ScaledSum | AllocateToReferencePerson + ): for item in items: if item.monetary is None: raise ValueError(f"Binding for {name!r}: {item.id} is not an amount.") @@ -784,8 +886,10 @@ def _check_transform_arity(binding: InputBinding) -> None: f"Binding for {name!r}: only float concepts scale; a scaled count " "would not decode to whole numbers." ) - if isinstance(transform, Sum) and len(items) < 2: + if isinstance(transform, Sum | ScaledSum) and len(items) < 2: raise ValueError(f"Binding for {name!r}: a sum reads two or more amounts.") + if isinstance(transform, ScaledSum) and len({item.entity for item in items}) != 1: + raise ValueError(f"Binding for {name!r}: a sum adds amounts on one entity.") if isinstance(transform, Product) and len(items) != 2: raise ValueError(f"Binding for {name!r}: a product reads two concepts.") if isinstance(transform, AllocateToReferencePerson) and items[0].entity != ( @@ -816,6 +920,13 @@ def _check_transform_arity(binding: InputBinding) -> None: f"Binding for {name!r}: role {transform.role.value} reads " f"{transform.concepts}." ) + if isinstance(transform, UnitComposition) and ( + binding.concepts != transform.concepts + ): + raise ValueError( + f"Binding for {name!r}: a unit composition reads what unit " + f"construction reads: {transform.concepts}." + ) if isinstance(transform, CoresidentChildCount) and ( binding.concepts != transform.concepts ): @@ -832,6 +943,13 @@ def _check_transform_arity(binding: InputBinding) -> None: def _check_group_rule(engine: str, binding: InputBinding) -> None: rule = binding.group_rule + if isinstance(binding.transform, UnitComposition): + if rule is not GroupRule.REFERENCE_MEMBER: + raise ValueError( + f"{engine}: {binding.engine_input!r} is a unit's own composition; " + "it takes the reference_member rule." + ) + return items = [concept(concept_id) for concept_id in binding.concepts] household_only = all(item.entity == "household" for item in items) amounts = all(item.monetary is not None for item in items) @@ -884,7 +1002,308 @@ class EncodedInputs: each carrying the frame's id columns plus the engine inputs that live on the engine's person or household entity. deferred: Bindings on engine group entities, which need engine unit - membership and were not executed. + membership and were not executed here; group encoding + (:meth:`ConceptMapping.encode_groups`) executes them on built + units. + """ + + tables: Mapping[str, pd.DataFrame] + deferred: tuple[InputBinding, ...] + + +class Allocation(StrEnum): + """Where an allocated household amount goes among the household's units.""" + + #: All of it on the unit that contains the household's reference person. + REFERENCE_UNIT = "reference_unit" + #: Shared among the household's units in proportion to their adults + #: (heads and partners); every unit has at least one. + PER_ADULT_SHARE = "per_adult_share" + + +@dataclass(frozen=True, eq=False) +class GroupMembership: + """An engine group entity's units, and who belongs to them. + + A unit-construction step builds these from the relationship pointers + (:func:`microcosm.frame.unit_construction.benefit_unit_membership`). + :meth:`ConceptMapping.encode_groups` checks them against the concept + frame before using them: membership aligned to the person index, integer + ids, members in the household their unit declares, one head per unit and + at most one partner, who is the head's partner by the pointers. + + Attributes: + entity: The frame group entity the units are (``family``). The unit + table carries ``_id``, ``_household_id`` and + ``_head_person_id``. + units: One row per unit. + person_unit: Each person's unit id, aligned to the person table. + person_role: Each person's :class:`UnitRole` value, aligned likewise. + """ + + entity: str + units: pd.DataFrame + person_unit: pd.Series + person_role: pd.Series + + @property + def id_column(self) -> str: + """The unit table's id column.""" + return f"{self.entity}_id" + + @property + def household_column(self) -> str: + """The unit table's column naming the household each unit nests in.""" + return f"{self.entity}_household_id" + + @property + def head_column(self) -> str: + """The unit table's column naming each unit's head.""" + return f"{self.entity}_head_person_id" + + +@dataclass(frozen=True, kw_only=True) +class GroupKnobs: + """Declared alternatives for how group inputs are encoded. + + Every knob names the engine inputs (or the state column) it changes, so + a scenario is data: a country pack translates its own knob names (rent + allocation, asset test, area column, rent factor) into these. A knob that + names an input the call does not produce is refused. + + Attributes: + allocation: Engine input -> where its household amount goes; inputs + not named here go to the reference unit. + zeroed: Numeric engine inputs set to zero (an asset test switched off + by giving every unit no assets). + factors: Numeric engine input -> a factor every unit's value is + multiplied by (a stock adjustment to rents). + state_columns: A state binding's declared column -> the column read + instead, both ``.`` (an alternative area + assignment). + """ + + allocation: Mapping[str, Allocation] = field(default_factory=dict) + zeroed: frozenset[str] = frozenset() + factors: Mapping[str, float] = field(default_factory=dict) + state_columns: Mapping[str, str] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__( + self, + "allocation", + MappingProxyType( + {name: Allocation(mode) for name, mode in self.allocation.items()} + ), + ) + object.__setattr__(self, "zeroed", frozenset(self.zeroed)) + factors = {} + for name, factor in self.factors.items(): + if isinstance(factor, bool) or not isinstance(factor, int | float): + raise ValueError(f"The factor for {name!r} must be a number.") + if not math.isfinite(factor) or factor < 0: + raise ValueError( + f"The factor for {name!r} must be finite and non-negative." + ) + factors[name] = float(factor) + object.__setattr__(self, "factors", MappingProxyType(factors)) + for declared, used in self.state_columns.items(): + for column in (declared, used): + _state_column(column) + if declared.split(".")[0] != used.split(".")[0]: + raise ValueError( + f"State column {declared!r} cannot be read from another " + f"entity's {used!r}." + ) + object.__setattr__( + self, "state_columns", MappingProxyType(dict(self.state_columns)) + ) + overlap = self.zeroed & set(self.factors) + if overlap: + raise ValueError(f"Inputs {sorted(overlap)} are both zeroed and scaled.") + + def targets(self) -> frozenset[str]: + """Every engine input a knob changes.""" + return frozenset(self.allocation) | self.zeroed | frozenset(self.factors) + + +class StateTest(StrEnum): + """How a state binding reads its model-state columns.""" + + #: True when any of the boolean columns is true. + ANY_TRUE = "any_true" + #: True when the one column equals the binding's value. + EQUALS = "equals" + + +#: The group rules a state binding may take, by the state's entity. +_STATE_RULES = { + "person": (GroupRule.ANY_MEMBER, GroupRule.REFERENCE_MEMBER), + "household": (GroupRule.HOUSEHOLD_VALUE,), +} + + +@dataclass(frozen=True, kw_only=True) +class StateBinding: + """A group engine input fed from model state rather than concepts. + + Model state is a column a graph node computed (a receipt flag assigned by + take-up, an area assigned by geography), not a survey fact, so it has no + concept and no place in a :class:`ConceptMapping`. The country pack + declares these with their evidence, and + :meth:`ConceptMapping.encode_groups` executes them alongside the + mapping's group bindings: the test gives a boolean per person or + household, the group rule collapses it onto the unit, and ``negate`` + flips the result (a unit with no recipient is a non-beneficiary unit). + + Attributes: + engine_input: The engine input's name. + engine_entity: The engine group entity the input lives on. + columns: The state columns read, each ``.`` on the + frame's ``person`` or ``household`` table (one entity). + test: How the columns give a boolean. + group_rule: How the boolean collapses onto the unit. + note: The evidence for the binding and any difference in meaning. + value: The value :attr:`StateTest.EQUALS` compares with. + negate: Whether the collapsed value is negated. + module: For module-scoped engines, the RuleSpec module. + canonical_input: For Axiom, the engine's canonical request name. + """ + + engine_input: str + engine_entity: str + columns: tuple[str, ...] + test: StateTest + group_rule: GroupRule + note: str + value: bool | int | str | None = None + negate: bool = False + module: str | None = None + canonical_input: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "test", StateTest(self.test)) + object.__setattr__(self, "group_rule", GroupRule(self.group_rule)) + object.__setattr__(self, "columns", tuple(self.columns)) + name = self.engine_input + if not name or not self.engine_entity: + raise ValueError("A state binding names its engine input and entity.") + if not self.note: + raise ValueError(f"State binding for {name!r} needs a note.") + if not self.columns or len(set(self.columns)) != len(self.columns): + raise ValueError(f"State binding for {name!r} reads distinct columns.") + entities = {_state_column(column)[0] for column in self.columns} + if len(entities) != 1: + raise ValueError(f"State binding for {name!r} reads columns on one entity.") + entity = entities.pop() + if self.group_rule not in _STATE_RULES[entity]: + allowed = [rule.value for rule in _STATE_RULES[entity]] + raise ValueError( + f"State binding for {name!r}: {entity} state collapses by " + f"{allowed}, not {self.group_rule.value!r}." + ) + if self.test is StateTest.EQUALS: + if len(self.columns) != 1 or self.value is None: + raise ValueError( + f"State binding for {name!r}: equals compares one column " + "with a value." + ) + if isinstance(self.value, float): + raise ValueError( + f"State binding for {name!r} compares with an exact value." + ) + elif self.value is not None: + raise ValueError(f"State binding for {name!r}: any_true takes no value.") + if not isinstance(self.negate, bool): + raise ValueError(f"State binding for {name!r}: negate is true or false.") + + @property + def ref(self) -> InputRef: + """The engine input this binding feeds.""" + return InputRef(self.engine_input, self.engine_entity, self.module) + + @property + def entity(self) -> str: + """The frame entity whose state the binding reads.""" + return _state_column(self.columns[0])[0] + + def to_dict(self) -> dict[str, object]: + """A JSON-ready form, omitting absent optional fields.""" + out: dict[str, object] = { + "engine_input": self.engine_input, + "engine_entity": self.engine_entity, + "columns": list(self.columns), + "test": self.test.value, + "group_rule": self.group_rule.value, + "note": self.note, + } + if self.value is not None: + out["value"] = self.value + if self.negate: + out["negate"] = True + if self.module is not None: + out["module"] = self.module + if self.canonical_input is not None: + out["canonical_input"] = self.canonical_input + return out + + @classmethod + def from_dict(cls, data: Mapping[str, object]) -> StateBinding: + """The state binding ``data`` describes (see :meth:`to_dict`). + + Raises: + ValueError: If ``data`` is not an object, or a field is missing, + unexpected or malformed. + """ + with _parsing("state binding"): + fields = _record_fields( + data, + "state binding", + required=( + "engine_input", + "engine_entity", + "columns", + "test", + "group_rule", + "note", + ), + optional=("value", "negate", "module", "canonical_input"), + ) + _text_fields( + fields, + "state binding", + ( + "engine_input", + "engine_entity", + "test", + "group_rule", + "note", + "module", + "canonical_input", + ), + ) + columns = fields["columns"] + if not isinstance(columns, list | tuple) or not all( + isinstance(column, str) for column in columns + ): + raise ValueError("A state binding's 'columns' must list columns.") + fields["columns"] = tuple(columns) + value = fields.get("value") + if value is not None and not isinstance(value, bool | int | str): + raise ValueError("A state binding's 'value' must be exact.") + return cls(**fields) + + +@dataclass(frozen=True, eq=False) +class EncodedGroupInputs: + """Engine group inputs computed from a concept frame and its units. + + Attributes: + tables: One table per frame group entity encoded (``family``), row + aligned to that membership's unit table, with the unit id column + plus the engine inputs. + deferred: Group bindings not executed: their engine entity has no + membership here, or they take the per-person ``person_role`` rule. """ tables: Mapping[str, pd.DataFrame] @@ -1010,6 +1429,11 @@ def _check_binding(self, binding: InputBinding) -> None: raise ValueError( f"{self.engine}: allocation targets the engine's person entity." ) + if not on_group and isinstance(binding.transform, UnitComposition): + raise ValueError( + f"{self.engine}: {binding.engine_input!r} tests a unit's " + "composition, so it lives on an engine group entity." + ) if on_group: _check_group_rule(self.engine, binding) @@ -1265,7 +1689,8 @@ def encode( Every executable binding whose concepts are all present in ``tables`` is computed. Bindings on engine group entities are returned as - deferred. Share parameters and take-up rates are needed only for the + deferred (:meth:`encode_groups` executes them on built units). Share + parameters and take-up rates are needed only for the bindings that use them. A take-up rate of ``None`` marks a program the caller leaves unseeded: its flag is not written, so the engine applies its own default. @@ -1305,6 +1730,201 @@ def encode( deferred=tuple(deferred), ) + def encode_groups( + self, + tables: Mapping[str, pd.DataFrame], + memberships: Mapping[str, GroupMembership], + *, + modules: Iterable[str] | None = None, + knobs: GroupKnobs | None = None, + state_bindings: Iterable[StateBinding] = (), + shares: Mapping[str, float] | None = None, + take_up_rates: Mapping[str, float] | None = None, + ) -> EncodedGroupInputs: + """Compute the engine group inputs a concept frame and its units supply. + + Every group binding whose engine entity has a membership, whose + module passes the filter and whose concepts are all present in + ``tables`` is executed by its :class:`GroupRule`: + + - ``sum_over_members``: the members' values summed; + - ``any_member``: true when any member's value is; + - ``reference_member``: the unit head's value (a + :class:`UnitComposition` is the unit's own); + - ``household_value``: the household's value on each of its units; + - ``allocate_to_reference_unit``: the household amount on the unit + that contains the household's reference person and zero on its + other units, or shared per adult when ``knobs`` says so; either + way each household's units sum back to its amount. + + ``person_role`` bindings, and bindings on an engine entity with no + membership, are returned as deferred. Each state binding is executed + the same way from its model-state columns. ``knobs`` then zero or + scale named inputs. + + Args: + tables: The concept frame's ``person`` and ``household`` tables, + carrying any model-state columns the state bindings read. + memberships: Engine group entity (``Family``) -> its units. + modules: The RuleSpec modules to encode for; ``None`` encodes all. + knobs: Declared encoding alternatives; ``None`` is the default. + state_bindings: Group inputs fed from model state. + shares: Share and fraction parameters, as for :meth:`encode`. + take_up_rates: Take-up rates, as for :meth:`encode`. + + Raises: + ValueError: If a membership does not match the frame (a person + outside every unit, a unit without exactly one head, members + in two households), a state binding feeds an input a concept + binding or another state binding also feeds, a knob names an + input this call does not produce or cannot change, or a value + a transform needs is missing. + """ + + knobs = knobs or GroupKnobs() + selected = None if modules is None else frozenset(modules) + shares = dict(shares or {}) + rates = dict(take_up_rates or {}) + person = tables["person"] + household = tables["household"] + context = _Context(person, household) + units: dict[str, _Units] = {} + for entity, membership in memberships.items(): + if not isinstance(membership, GroupMembership): + raise ValueError(f"The {entity!r} membership is not a GroupMembership.") + if any( + group.membership.entity == membership.entity for group in units.values() + ): + raise ValueError( + f"Two engine entities name the frame entity {membership.entity!r}." + ) + units[entity] = _Units(membership, context) + out = { + group.membership.entity: group.membership.units.loc[ + :, [group.membership.id_column] + ].reset_index(drop=True) + for group in units.values() + } + # One encoded column per (frame entity, name): an input name shared by + # several modules is computed once, as _check_shared_names requires. + produced: dict[str, set[str]] = defaultdict(set) + allocated: set[str] = set() + deferred: list[InputBinding] = [] + concept_names = { + (binding.engine_input, binding.engine_entity) + for binding in self.bindings + if binding.group_rule is not None + } + + def keep(module: str | None) -> bool: + return selected is None or module in selected + + def write(entity: str, name: str, values: np.ndarray) -> None: + frame_entity = units[entity].membership.entity + out[frame_entity][name] = values + produced[name].add(frame_entity) + + for binding in self.bindings: + if binding.group_rule is None or not keep(binding.module): + continue + group = units.get(binding.engine_entity) + if group is None or binding.group_rule is GroupRule.PERSON_ROLE: + deferred.append(binding) + continue + if not all(context.has(concept_id) for concept_id in binding.concepts): + continue + if ( + isinstance(binding.transform, TakeUpThreshold) + and binding.transform.program in rates + and rates[binding.transform.program] is None + ): + continue + if binding.group_rule is GroupRule.ALLOCATE_TO_REFERENCE_UNIT: + allocated.add(binding.engine_input) + mode = knobs.allocation.get(binding.engine_input, Allocation.REFERENCE_UNIT) + write( + binding.engine_entity, + binding.engine_input, + _group_values(binding, context, group, mode, shares, rates), + ) + + seen_state: dict[tuple[str, str], tuple[InputRef, dict[str, object]]] = {} + for state in state_bindings: + if not keep(state.module): + continue + key = (state.engine_input, state.engine_entity) + if key in concept_names: + raise ValueError( + f"{state.ref.label()} is bound by a concept and by state " + "(one encoded column per input name)." + ) + signature = { + name: value + for name, value in state.to_dict().items() + if name not in ("module", "canonical_input") + } + if key in seen_state: + first, first_signature = seen_state[key] + if first == state.ref: + raise ValueError(f"{state.ref.label()} is bound twice by state.") + if first_signature != signature: + raise ValueError( + f"{state.engine_input!r} on {state.engine_entity!r} is " + f"computed differently in {first.module} and {state.module}." + ) + continue + seen_state[key] = (state.ref, signature) + group = units.get(state.engine_entity) + if group is None: + raise ValueError( + f"State binding {state.engine_input!r} needs a " + f"{state.engine_entity!r} membership." + ) + write( + state.engine_entity, + state.engine_input, + _state_values(state, context, group, knobs.state_columns), + ) + + unknown = sorted(knobs.targets() - set(produced)) + if unknown: + raise ValueError( + f"Knobs name inputs this encoding did not produce: {unknown}." + ) + ambiguous = sorted(name for name in knobs.targets() if len(produced[name]) > 1) + if ambiguous: + raise ValueError( + f"Knob targets {ambiguous} live on several group entities." + ) + not_allocated = sorted(set(knobs.allocation) - allocated) + if not_allocated: + raise ValueError( + f"{not_allocated} are not allocated; no allocation knob applies to them." + ) + read = { + column + for state in state_bindings + if keep(state.module) + for column in state.columns + } + stray = sorted(set(knobs.state_columns) - read) + if stray: + raise ValueError(f"Knobs replace state columns no binding reads: {stray}.") + for name in sorted(knobs.zeroed | set(knobs.factors)): + (frame_entity,) = produced[name] + table = out[frame_entity] + values = table[name].to_numpy() + if values.dtype.kind not in "iuf": + raise ValueError(f"Knob target {name!r} is not numeric.") + if name in knobs.zeroed: + table[name] = np.zeros(len(values), dtype=values.dtype) + else: + table[name] = values.astype(np.float64) * knobs.factors[name] + return EncodedGroupInputs( + tables=MappingProxyType(out), + deferred=tuple(deferred), + ) + def decode(self, tables: Mapping[str, pd.DataFrame]) -> dict[str, pd.DataFrame]: """Recover every invertible concept present in engine input tables. @@ -1420,6 +2040,20 @@ def _apply( ], axis=0, ) + if isinstance(transform, ScaledSum): + total = np.sum( + [ + context.values(concept_id).to_numpy(dtype=np.float64) + for concept_id in concepts + ], + axis=0, + ) + return total * transform.factor + if isinstance(transform, UnitComposition): + raise ValueError( + f"{binding.engine_input!r} tests unit composition; it needs unit " + "membership (encode_groups)." + ) if isinstance(transform, Product): first, second = ( context.values(concept_id).to_numpy(dtype=np.float64) @@ -1506,6 +2140,244 @@ def _child_counts(context: _Context, max_age: int | None) -> np.ndarray: return counts +def _state_column(column: str) -> tuple[str, str]: + """``.`` split, for a person or household column.""" + entity, _, name = column.partition(".") if isinstance(column, str) else ("", "", "") + if entity not in CONCEPT_ENTITIES or not name or "." in name: + raise ValueError( + f"A state column is 'person.' or 'household.', " + f"not {column!r}." + ) + return entity, name + + +class _Units: + """A membership checked against a concept frame, as row positions.""" + + def __init__(self, membership: GroupMembership, context: _Context) -> None: + self.membership = membership + units = membership.units + columns = ( + membership.id_column, + membership.household_column, + membership.head_column, + ) + missing = [name for name in columns if name not in units.columns] + if missing: + raise ValueError(f"A {membership.entity} unit table lacks {missing}.") + person = context.person + for series in (membership.person_unit, membership.person_role): + if not isinstance(series, pd.Series) or not series.index.equals( + person.index + ): + raise ValueError( + f"{membership.entity} membership must align to the person " + "table's index." + ) + if membership.person_unit.isna().any(): + raise ValueError(f"Every person needs a {membership.entity}.") + for name, values in ( + ("person unit ids", membership.person_unit), + *((column, units[column]) for column in columns), + ): + if values.dtype.kind not in "iu" and not pd.api.types.is_integer_dtype( + values.dtype + ): + raise ValueError( + f"{membership.entity} {name} must be integers, not {values.dtype}." + ) + ids = units[membership.id_column].to_numpy(dtype=np.int64) + if len(set(ids.tolist())) != len(ids): + raise ValueError(f"{membership.entity} ids must be unique.") + rows = pd.Index(ids).get_indexer( + membership.person_unit.to_numpy(dtype=np.int64) + ) + if (rows < 0).any(): + raise ValueError( + f"{int((rows < 0).sum())} person(s) name an unknown " + f"{membership.entity}." + ) + roles = membership.person_role.astype(object).to_numpy() + known = {role.value for role in UnitRole} + if not all(role in known for role in roles.tolist()): + raise ValueError(f"Unit roles come from {sorted(known)}.") + n = len(units) + if (np.bincount(rows, minlength=n) == 0).any(): + raise ValueError(f"Every {membership.entity} needs a member.") + is_head = roles == UnitRole.HEAD.value + if not (np.bincount(rows[is_head], minlength=n) == 1).all(): + raise ValueError(f"Every {membership.entity} needs exactly one head.") + if (np.bincount(rows[roles == UnitRole.PARTNER.value], minlength=n) > 1).any(): + raise ValueError(f"A {membership.entity} has at most one partner.") + head_rows = np.empty(n, dtype=np.int64) + head_rows[rows[is_head]] = np.flatnonzero(is_head) + person_ids = person[_PERSON_ID].to_numpy(dtype=np.int64) + if not np.array_equal( + person_ids[head_rows], units[membership.head_column].to_numpy(np.int64) + ): + raise ValueError( + f"Each {membership.entity}'s head column must name its head member." + ) + if "partner_person_id" in person.columns: + partner = context.person_rows(person["partner_person_id"]) + head_of = head_rows[rows] + is_partner = roles == UnitRole.PARTNER.value + if (partner[is_partner] != head_of[is_partner]).any(): + raise ValueError( + f"A {membership.entity} partner must be its head's partner." + ) + heads = np.flatnonzero(is_head) + mate = partner[heads] + together = mate >= 0 + together[together] = rows[mate[together]] == rows[heads[together]] + if (roles[mate[together]] != UnitRole.PARTNER.value).any(): + raise ValueError( + f"A head's partner in the same {membership.entity} must be " + "its partner." + ) + unit_households = units[membership.household_column].to_numpy(np.int64) + person_households = person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64) + if not np.array_equal(person_households, unit_households[rows]): + raise ValueError( + f"Every {membership.entity} must nest in its declared household." + ) + household_rows = pd.Index( + context.household[_HOUSEHOLD_ID].to_numpy() + ).get_indexer(unit_households) + if (household_rows < 0).any(): + raise ValueError(f"A {membership.entity} names an unknown household.") + self.rows = rows + self.roles = roles + self.head_rows = head_rows + self.household_rows = household_rows + self.is_adult = roles != UnitRole.DEPENDENT_CHILD.value + self.n = n + + def count(self, role: UnitRole) -> np.ndarray: + """How many members of each unit hold ``role``.""" + return np.bincount(self.rows[self.roles == role.value], minlength=self.n) + + def collapse(self, rule: GroupRule, values: np.ndarray) -> np.ndarray: + """Person values collapsed onto units by a person-reading rule.""" + if rule is GroupRule.REFERENCE_MEMBER: + return values[self.head_rows] + if rule is GroupRule.ANY_MEMBER: + flags = np.zeros(self.n, dtype=bool) + np.logical_or.at(flags, self.rows, np.asarray(values, dtype=bool)) + return flags + if rule is GroupRule.SUM_OVER_MEMBERS: + values = np.asarray(values) + kind = np.int64 if values.dtype.kind in "biu" else np.float64 + totals = np.zeros(self.n, dtype=kind) + np.add.at(totals, self.rows, values.astype(kind)) + return totals + raise TypeError( + f"Rule {rule} does not collapse person values." + ) # pragma: no cover + + def reference_units(self, context: _Context) -> np.ndarray: + """Whether each unit contains its household's reference person.""" + _require(context, _REFERENCE_PERSON, "Allocating to the reference unit") + references = context.person_rows(context.household["reference_person_id"]) + households = self.household_rows + reference = references[households] + if (reference < 0).any(): + raise ValueError("A household with units has no reference person.") + own = context.person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64) + declared = context.household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64) + if not np.array_equal(own[reference], declared[households]): + raise ValueError("A household's reference person must be its member.") + return self.rows[reference] == np.arange(self.n) + + +def _group_values( + binding: InputBinding, + context: _Context, + group: _Units, + mode: Allocation, + shares: Mapping[str, float], + rates: Mapping[str, float], +) -> np.ndarray: + rule = binding.group_rule + transform = binding.transform + if isinstance(transform, UnitComposition): + partners = group.count(UnitRole.PARTNER) + children = group.count(UnitRole.DEPENDENT_CHILD) + feature = transform.feature + if feature is UnitFeature.HAS_PARTNER: + return partners > 0 + if feature is UnitFeature.HAS_DEPENDENT_CHILD: + return children >= 1 + if feature is UnitFeature.TWO_OR_MORE_DEPENDENT_CHILDREN: + return children >= 2 + return (partners == 0) & (children >= 1) + values = _apply(binding, context, shares, rates) + if rule is GroupRule.HOUSEHOLD_VALUE: + return np.asarray(values)[group.household_rows] + if rule is GroupRule.ALLOCATE_TO_REFERENCE_UNIT: + amounts = np.asarray(values, dtype=np.float64)[group.household_rows] + if mode is Allocation.REFERENCE_UNIT: + return np.where(group.reference_units(context), amounts, 0.0) + adults = np.bincount( + group.rows, weights=group.is_adult.astype(np.float64), minlength=group.n + ) + per_household = np.bincount( + group.household_rows, weights=adults, minlength=len(context.household) + ) + return amounts * adults / per_household[group.household_rows] + return group.collapse(rule, np.asarray(values)) + + +def _state_values( + state: StateBinding, + context: _Context, + group: _Units, + replaced: Mapping[str, str], +) -> np.ndarray: + table = context.table(state.entity) + columns = [] + for declared in state.columns: + _, name = _state_column(replaced.get(declared, declared)) + if name not in table.columns: + raise ValueError( + f"State binding {state.engine_input!r} needs the {state.entity} " + f"column {name!r}." + ) + values = table[name] + if values.isna().any(): + raise ValueError(f"State column {name!r} has missing values.") + columns.append(values) + if state.test is StateTest.ANY_TRUE: + flags = np.zeros(len(table), dtype=bool) + for values in columns: + if values.dtype.kind != "b": + raise ValueError( + f"State column {values.name!r} must be boolean for any_true." + ) + flags |= values.to_numpy(dtype=bool) + else: + values = columns[0] + kind = values.dtype.kind + expected = ( + kind == "b" + if isinstance(state.value, bool) + else kind in "iu" + if isinstance(state.value, int) + else kind in "OUT" or isinstance(values.dtype, pd.StringDtype) + ) + if not expected: + raise ValueError( + f"State column {values.name!r} ({values.dtype}) cannot equal " + f"{state.value!r}." + ) + flags = values.astype(object).eq(state.value).to_numpy(dtype=bool) + if state.group_rule is GroupRule.HOUSEHOLD_VALUE: + result = flags[group.household_rows] + else: + result = group.collapse(state.group_rule, flags) + return ~result if state.negate else result + + def _invert( item: Concept, decoder: tuple[InputBinding, ...], diff --git a/packages/microcosm-frame/src/microcosm/frame/concepts.py b/packages/microcosm-frame/src/microcosm/frame/concepts.py index f048ada835..117d5fdf00 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concepts.py +++ b/packages/microcosm-frame/src/microcosm/frame/concepts.py @@ -122,7 +122,8 @@ #: units, benefit units, SPM units, families) are engine constructs built #: from relationships. Today the US operator in microcosm.frame.units builds #: them from raw CPS roster columns and the UK adapter requires them already -#: present; building them from these concepts' pointers is future work. +#: present; microcosm.frame.unit_construction builds benefit units from these +#: concepts' pointers. CONCEPT_ENTITIES: tuple[str, ...] = ("person", "household") #: A concept frame's entity structure, in the kernel's id conventions. diff --git a/packages/microcosm-frame/src/microcosm/frame/input_closure.py b/packages/microcosm-frame/src/microcosm/frame/input_closure.py new file mode 100644 index 0000000000..7e7560dfb6 --- /dev/null +++ b/packages/microcosm-frame/src/microcosm/frame/input_closure.py @@ -0,0 +1,850 @@ +"""How every root input of a bound Axiom module is fed: the input closure. + +An Axiom module reads every name a rule references and does not derive. The +adapter (:mod:`microcosm.frame.adapters.axiom`) omits an input the frame does +not carry, and the engine fails a request whose evaluation reaches an absent +input. A concept mapping binds only the inputs a concept determines, so a run +also needs a decision for every other root input. An :class:`InputClosure` is +that decision, as data: for each bound module it puts every root input of the +module's committed input surface in exactly one class. + +- ``encoded``: computed per record, by a concept binding of the mapping + (:class:`~microcosm.frame.concept_mapping.InputBinding`), by a + :class:`~microcosm.frame.concept_mapping.StateBinding` from model state, or + by another graph node (``graph_node``, for example an engine-computed + bridge value). +- ``defaulted``: one value for every record, with the reason and the + provision it stands in for. Defaults are modelling content, so each says + whether it is a new default for review. +- ``engine_optional``: the engine itself supplies the input when it is + absent. The entry carries the evidence that the engine does. + +The closure also declares the encoding knobs a country's scenarios turn +(:meth:`InputClosure.group_knobs` translates their values into +:class:`~microcosm.frame.concept_mapping.GroupKnobs`) and how an +undetermined judgment (code ``0``) reaches take-up +(:func:`resolve_judgments`). :meth:`InputClosure.check` compares a closure +with the mapping and the engine-generated surface: the classes partition each +module's surface exactly, and every concept-encoded entry is the mapping's +binding and nothing else. +""" + +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from enum import StrEnum +from types import MappingProxyType +from typing import TYPE_CHECKING + +import numpy as np +import pandas as pd + +from microcosm.frame.concept_mapping import ( + Allocation, + ConceptMapping, + GroupKnobs, + InputRef, + StateBinding, +) +from microcosm.frame.concepts import ( + ContentBasis, + TransportRule, + _parsing, + _record_fields, + _text_fields, + concept, +) + +if TYPE_CHECKING: + from microcosm.frame.adapters.axiom_input_surface import AxiomInputSurface + +__all__ = [ + "INPUT_CLOSURE_FORMAT", + "ClosureClass", + "ClosureEntry", + "EncodedBy", + "InputClosure", + "Knob", + "KnobKind", + "UndeterminedAction", + "UndeterminedPolicy", + "apply_defaults", + "resolve_judgments", +] + +#: The serialized closure format :meth:`InputClosure.to_dict` writes. +INPUT_CLOSURE_FORMAT = "microcosm.axiom_input_closure.v1" + + +class ClosureClass(StrEnum): + """How one root input is fed.""" + + ENCODED = "encoded" + DEFAULTED = "defaulted" + ENGINE_OPTIONAL = "engine_optional" + + +class EncodedBy(StrEnum): + """What computes an encoded input.""" + + #: A binding of the concept mapping. + CONCEPT_BINDING = "concept_binding" + #: A state binding the closure declares. + STATE_BINDING = "state_binding" + #: Another graph node, named by the entry. + GRAPH_NODE = "graph_node" + + +class UndeterminedAction(StrEnum): + """What take-up does with an undetermined (code ``0``) judgment.""" + + #: Fail: an undetermined judgment stops the run. + REFUSE = "refuse" + #: Treat it as not holding. + NOT_ELIGIBLE = "not_eligible" + #: Treat it as holding. + ELIGIBLE = "eligible" + + +class KnobKind(StrEnum): + """What a declared knob changes.""" + + #: Where allocated household amounts go (an :class:`Allocation` value). + ALLOCATION = "allocation" + #: A test switched on (``true``) or off; off zeroes the targets. + SWITCH_ZEROES = "switch_zeroes" + #: Which column a state binding reads (a column on the same entity). + STATE_COLUMN = "state_column" + #: A non-negative factor on the targets. + FACTOR = "factor" + + +_JSON_SCALAR = bool | int | float | str + + +@dataclass(frozen=True, kw_only=True) +class Knob: + """One declared encoding alternative a scenario can turn. + + Attributes: + name: The knob's name in the country's scenario list. + kind: What it changes. + targets: The engine inputs it changes, or for + :attr:`KnobKind.STATE_COLUMN` the one declared state column + (``.``) it replaces. + central: The value the central run uses. + values: The values it may take (allocation modes or column names); + empty for switches and factors. + method_card_row: The method-card row that rules on it. + note: What turning it means. + """ + + name: str + kind: KnobKind + targets: tuple[str, ...] + central: _JSON_SCALAR + values: tuple[_JSON_SCALAR, ...] + method_card_row: str + note: str + + def __post_init__(self) -> None: + object.__setattr__(self, "kind", KnobKind(self.kind)) + object.__setattr__(self, "targets", tuple(self.targets)) + object.__setattr__(self, "values", tuple(self.values)) + if not self.name or not self.note or not self.method_card_row: + raise ValueError("A knob needs a name, a note and a method-card row.") + if not self.targets or len(set(self.targets)) != len(self.targets): + raise ValueError(f"Knob {self.name!r} names distinct targets.") + if self.kind is KnobKind.STATE_COLUMN and len(self.targets) != 1: + raise ValueError(f"Knob {self.name!r} replaces one state column.") + if self.kind in (KnobKind.ALLOCATION, KnobKind.STATE_COLUMN): + if not self.values: + raise ValueError(f"Knob {self.name!r} lists the values it takes.") + elif self.values: + raise ValueError(f"Knob {self.name!r} takes no value list.") + self.resolve(self.central) + + def resolve(self, value: object) -> _JSON_SCALAR: + """``value`` checked against the knob's kind and allowed values.""" + kind = self.kind + if kind is KnobKind.ALLOCATION or kind is KnobKind.STATE_COLUMN: + if value not in self.values: + raise ValueError( + f"Knob {self.name!r} takes {list(self.values)}, not {value!r}." + ) + if kind is KnobKind.ALLOCATION: + Allocation(value) + return value + if kind is KnobKind.SWITCH_ZEROES: + if not isinstance(value, bool): + raise ValueError(f"Knob {self.name!r} is switched true or false.") + return value + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError(f"Knob {self.name!r} takes a number.") + if not math.isfinite(value) or value < 0: + raise ValueError(f"Knob {self.name!r} takes a finite, non-negative factor.") + return float(value) + + def to_dict(self) -> dict[str, object]: + """A JSON-ready form.""" + return { + "name": self.name, + "kind": self.kind.value, + "targets": list(self.targets), + "central": self.central, + "values": list(self.values), + "method_card_row": self.method_card_row, + "note": self.note, + } + + @classmethod + def from_dict(cls, data: Mapping[str, object]) -> Knob: + """The knob ``data`` describes (see :meth:`to_dict`).""" + with _parsing("knob"): + fields = _record_fields( + data, + "knob", + required=( + "name", + "kind", + "targets", + "central", + "values", + "method_card_row", + "note", + ), + ) + _text_fields(fields, "knob", ("name", "kind", "method_card_row", "note")) + for name in ("targets", "values"): + if not isinstance(fields[name], list | tuple): + raise ValueError(f"A knob's {name!r} must be a list.") + if not all(isinstance(target, str) for target in fields["targets"]): + raise ValueError("A knob's 'targets' must name inputs or a column.") + return cls(**fields) + + +@dataclass(frozen=True, kw_only=True) +class UndeterminedPolicy: + """How an undetermined entitlement judgment reaches take-up. + + Attributes: + action: What take-up does with code ``0``. + reason: Why, with the engine evidence. + alternatives: The other actions a reviewer could choose. + new_default: Whether this is a new modelling default for review. + """ + + action: UndeterminedAction + reason: str + alternatives: tuple[UndeterminedAction, ...] + new_default: bool + + def __post_init__(self) -> None: + object.__setattr__(self, "action", UndeterminedAction(self.action)) + object.__setattr__( + self, + "alternatives", + tuple(UndeterminedAction(item) for item in self.alternatives), + ) + if not self.reason: + raise ValueError("An undetermined-judgment policy needs a reason.") + if self.action in self.alternatives: + raise ValueError("The chosen action is not its own alternative.") + if not isinstance(self.new_default, bool): + raise ValueError("new_default is true or false.") + + def to_dict(self) -> dict[str, object]: + """A JSON-ready form.""" + return { + "action": self.action.value, + "reason": self.reason, + "alternatives": [item.value for item in self.alternatives], + "new_default": self.new_default, + } + + +@dataclass(frozen=True, kw_only=True) +class ClosureEntry: + """How one root input of one module is fed. + + Attributes: + module: The RuleSpec module, relative to its rulespec root. + entity: The engine entity whose program reads the input. + input: The root input's name. + closure_class: Its class (serialized as ``class``). + encoded_by: For an encoded input, what computes it. + state_binding: For a state-encoded input, the binding. + node: For a graph-node input, the node id that owns it. + value: For a defaulted input, the value every record gets. + reason: Why the input is fed this way. Required except for a concept + binding, whose evidence is the binding's own note. + citation: For a defaulted input, the provision the default stands in + for. + new_default: For a defaulted input, whether it is a new modelling + default for review. + awaiting: For a concept-encoded input whose binding lands in a named + later change, that change; the mapping may lack the binding until + then, and a closure whose awaited binding exists is out of date. + """ + + module: str + entity: str + input: str + closure_class: ClosureClass + encoded_by: EncodedBy | None = None + state_binding: StateBinding | None = None + node: str | None = None + value: bool | int | float | None = None + reason: str = "" + citation: str | None = None + new_default: bool = False + awaiting: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "closure_class", ClosureClass(self.closure_class)) + if self.encoded_by is not None: + object.__setattr__(self, "encoded_by", EncodedBy(self.encoded_by)) + label = f"{self.input} ({self.entity}, {self.module})" + if not self.module or not self.entity or not self.input: + raise ValueError("A closure entry names its module, entity and input.") + encoded = self.closure_class is ClosureClass.ENCODED + if encoded != (self.encoded_by is not None): + raise ValueError(f"{label}: exactly an encoded input says what encodes it.") + if (self.encoded_by is EncodedBy.STATE_BINDING) != ( + self.state_binding is not None + ): + raise ValueError(f"{label}: a state-encoded input carries its binding.") + if self.state_binding is not None and self.state_binding.ref != self.ref: + raise ValueError(f"{label}: its state binding feeds another input.") + if (self.encoded_by is EncodedBy.GRAPH_NODE) != bool(self.node): + raise ValueError(f"{label}: a graph-node input names its node.") + defaulted = self.closure_class is ClosureClass.DEFAULTED + if defaulted: + value = self.value + if value is None or not isinstance(value, bool | int | float): + raise ValueError(f"{label}: a default is a boolean or a number.") + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"{label}: a default is finite.") + if not self.citation: + raise ValueError(f"{label}: a default cites the provision it fills.") + elif self.value is not None or self.citation is not None or self.new_default: + raise ValueError(f"{label}: only a default carries a value or citation.") + if not isinstance(self.new_default, bool): + raise ValueError(f"{label}: new_default is true or false.") + if self.encoded_by is not EncodedBy.CONCEPT_BINDING and not self.reason: + raise ValueError(f"{label}: needs a reason.") + if self.awaiting is not None and ( + self.encoded_by is not EncodedBy.CONCEPT_BINDING or not self.awaiting + ): + raise ValueError(f"{label}: only a concept binding awaits a change.") + + @property + def ref(self) -> InputRef: + """The input this entry closes.""" + return InputRef(self.input, self.entity, self.module) + + def to_dict(self) -> dict[str, object]: + """A JSON-ready form, omitting absent optional fields.""" + out: dict[str, object] = { + "rulespec_path": self.module, + "entity": self.entity, + "input": self.input, + "class": self.closure_class.value, + } + if self.encoded_by is not None: + out["encoded_by"] = self.encoded_by.value + if self.state_binding is not None: + # The entry names the module once; a country package's spec keys + # never say "module" (they say rulespec_path). + state = self.state_binding.to_dict() + state.pop("module", None) + out["state_binding"] = state + if self.node is not None: + out["node"] = self.node + if self.value is not None: + out["value"] = self.value + if self.reason: + out["reason"] = self.reason + if self.citation is not None: + out["citation"] = self.citation + if self.new_default: + out["new_default"] = True + if self.awaiting is not None: + out["awaiting"] = self.awaiting + return out + + @classmethod + def from_dict(cls, data: Mapping[str, object]) -> ClosureEntry: + """The entry ``data`` describes (see :meth:`to_dict`).""" + with _parsing("closure entry"): + fields = _record_fields( + data, + "closure entry", + required=("rulespec_path", "entity", "input", "class"), + optional=( + "encoded_by", + "state_binding", + "node", + "value", + "reason", + "citation", + "new_default", + "awaiting", + ), + ) + _text_fields( + fields, + "closure entry", + ( + "rulespec_path", + "entity", + "input", + "class", + "encoded_by", + "node", + "reason", + "citation", + "awaiting", + ), + ) + fields["closure_class"] = fields.pop("class") + fields["module"] = fields.pop("rulespec_path") + if "state_binding" in fields: + state = fields["state_binding"] + if not isinstance(state, Mapping) or "module" in state: + raise ValueError( + "A closure entry's state binding is an object that takes " + "its module from the entry's rulespec_path." + ) + fields["state_binding"] = StateBinding.from_dict( + {**state, "module": fields["module"]} + ) + return cls(**fields) + + +@dataclass(frozen=True, kw_only=True) +class InputClosure: + """Every root input of a country's bound Axiom modules, closed. + + Attributes: + country: The rulespec country code. + content_basis: The frames the closure feeds. On a ``transport`` frame + the concepts transport drops (donor program receipts) are absent, + so a binding that reads one never runs and its input must be + closed some other way. + mapping_engine: The concept mapping the closure was reviewed with. + mapping_engine_version: That mapping's ``engine_version``. + surface_rulespec_commit: The rulespec commit the input surface was + generated from. + surface_engine_commit: The engine commit that generated it. + modules: Module path -> SHA-256 of the module bytes reviewed. + engine_optional_evidence: What the engine does with an absent input, + with its locators; the evidence any ``engine_optional`` entry + rests on, and the reason there may be none. + undetermined: How undetermined judgments reach take-up. + knobs: The encoding knobs scenarios turn. + entries: One entry per root input of every listed module. + """ + + country: str + content_basis: ContentBasis + mapping_engine: str + mapping_engine_version: str + surface_rulespec_commit: str + surface_engine_commit: str + modules: Mapping[str, str] + engine_optional_evidence: str + undetermined: UndeterminedPolicy + knobs: tuple[Knob, ...] + entries: tuple[ClosureEntry, ...] + + def __post_init__(self) -> None: + object.__setattr__(self, "content_basis", ContentBasis(self.content_basis)) + object.__setattr__(self, "modules", MappingProxyType(dict(self.modules))) + object.__setattr__(self, "knobs", tuple(self.knobs)) + object.__setattr__(self, "entries", tuple(self.entries)) + if not self.engine_optional_evidence: + raise ValueError( + "A closure states what the engine does with absent inputs." + ) + twice = [ + ref for ref, n in Counter(e.ref for e in self.entries).items() if n > 1 + ] + if twice: + raise ValueError(f"Inputs closed twice: {[ref.label() for ref in twice]}.") + unlisted = sorted({entry.module for entry in self.entries} - set(self.modules)) + if unlisted: + raise ValueError( + f"Entries name modules the closure does not list: {unlisted}." + ) + names = [knob.name for knob in self.knobs] + if len(set(names)) != len(names): + raise ValueError("Knob names must be unique.") + encoded = { + entry.input + for entry in self.entries + if entry.closure_class is ClosureClass.ENCODED + } + state_columns = { + column + for entry in self.entries + if entry.state_binding is not None + for column in entry.state_binding.columns + } + for knob in self.knobs: + if knob.kind is KnobKind.STATE_COLUMN: + missing = set(knob.targets) - state_columns + else: + missing = set(knob.targets) - encoded + if missing: + raise ValueError( + f"Knob {knob.name!r} targets {sorted(missing)}, which no " + "encoded entry provides." + ) + + # --- Queries ----------------------------------------------------------- + + def entries_for(self, module: str) -> tuple[ClosureEntry, ...]: + """The entries of one module.""" + return tuple(entry for entry in self.entries if entry.module == module) + + def defaults(self, module: str, entity: str) -> dict[str, bool | int | float]: + """Input name -> default, for one module's engine entity.""" + return { + entry.input: entry.value + for entry in self.entries_for(module) + if entry.entity == entity and entry.closure_class is ClosureClass.DEFAULTED + } + + def state_bindings( + self, modules: Iterable[str] | None = None + ) -> tuple[StateBinding, ...]: + """The declared state bindings, optionally for some modules only.""" + selected = None if modules is None else frozenset(modules) + return tuple( + entry.state_binding + for entry in self.entries + if entry.state_binding is not None + and (selected is None or entry.module in selected) + ) + + def graph_node_inputs(self) -> dict[InputRef, str]: + """Input -> the graph node that supplies it.""" + return { + entry.ref: entry.node + for entry in self.entries + if entry.encoded_by is EncodedBy.GRAPH_NODE + } + + def new_defaults(self) -> tuple[ClosureEntry, ...]: + """The defaults marked new: the closure's part of a method review.""" + return tuple(entry for entry in self.entries if entry.new_default) + + def group_knobs(self, values: Mapping[str, object] | None = None) -> GroupKnobs: + """The encoding a scenario's knob values give. + + Knobs not in ``values`` take their central value. + + Raises: + ValueError: If ``values`` names an unknown knob or a value the + knob does not take. + """ + values = dict(values or {}) + known = {knob.name for knob in self.knobs} + unknown = sorted(set(values) - known) + if unknown: + raise ValueError( + f"Unknown knobs {unknown}; the closure declares {sorted(known)}." + ) + allocation: dict[str, Allocation] = {} + zeroed: set[str] = set() + factors: dict[str, float] = {} + state_columns: dict[str, str] = {} + for knob in self.knobs: + value = knob.resolve(values.get(knob.name, knob.central)) + # A knob at a value that changes nothing names no target, so the + # central encoding of a module subset is never refused for it. + if knob.kind is KnobKind.ALLOCATION: + if Allocation(value) is not Allocation.REFERENCE_UNIT: + allocation.update(dict.fromkeys(knob.targets, Allocation(value))) + elif knob.kind is KnobKind.SWITCH_ZEROES: + if not value: + zeroed.update(knob.targets) + elif knob.kind is KnobKind.FACTOR: + for target in knob.targets: + factors[target] = factors.get(target, 1.0) * value + factors = {name: f for name, f in factors.items() if f != 1.0} + else: + (declared,) = knob.targets + entity = declared.partition(".")[0] + replacement = f"{entity}.{value}" + if replacement != declared: + state_columns[declared] = replacement + return GroupKnobs( + allocation=allocation, + zeroed=frozenset(zeroed), + factors=factors, + state_columns=state_columns, + ) + + # --- Checks ------------------------------------------------------------ + + def check( + self, mapping: ConceptMapping, surface: AxiomInputSurface + ) -> tuple[str, ...]: + """Every way the closure disagrees with the mapping or the surface. + + - Each listed module is in the surface, compiled, with the reviewed + SHA-256, and the mapping names the reviewed engine version. + - The entries of a module are exactly its surface's root inputs, and + every input the mapping binds in a listed module is closed. + - An input is concept-encoded exactly when the mapping binds it with + a binding that runs on the closure's content basis (on a transport + frame, one that reads no concept transport drops), or when its + entry awaits the binding; an awaited binding that exists is + reported, so the closure is updated when it lands. + + Args: + mapping: The country's concept mapping. + surface: The engine-generated input surface. + + Returns: + One message per disagreement; empty when the closure holds. + """ + problems: list[str] = [] + if mapping.engine != self.mapping_engine: + problems.append( + f"Closure is for {self.mapping_engine}, not {mapping.engine}." + ) + if mapping.engine_version != self.mapping_engine_version: + problems.append( + f"Mapping engine_version {mapping.engine_version} is not the " + f"reviewed {self.mapping_engine_version}." + ) + if surface.rulespec_commit != self.surface_rulespec_commit: + problems.append( + f"Surface rulespec {surface.rulespec_commit} is not " + f"{self.surface_rulespec_commit}." + ) + if surface.engine_commit != self.surface_engine_commit: + problems.append( + f"Surface engine {surface.engine_commit} is not {self.surface_engine_commit}." + ) + bound = { + binding.ref: binding + for binding in mapping.bindings + if binding.module in self.modules + } + runs = {ref for ref, binding in bound.items() if self._runs(binding)} + closed_refs = {entry.ref for entry in self.entries} + for ref in sorted(set(bound) - closed_refs): + problems.append(f"{ref.label()} is bound by the mapping but not closed.") + for path, sha256 in self.modules.items(): + module = surface.modules.get(path) + if module is None: + problems.append(f"{path} is not in the input surface.") + continue + if module.status != "compiled": + problems.append(f"{path} did not compile ({module.status}).") + if module.sha256 != sha256: + problems.append(f"{path} has SHA-256 {module.sha256}, not {sha256}.") + expected = set(module.refs()) + closed = {entry.ref for entry in self.entries_for(path)} + for ref in sorted(expected - closed): + problems.append(f"{ref.label()} is not closed.") + for ref in sorted(closed - expected): + problems.append(f"{ref.label()} is closed but not a root input.") + for entry in self.entries: + ref = entry.ref + by_concept = entry.encoded_by is EncodedBy.CONCEPT_BINDING + if entry.awaiting is not None: + if ref in bound: + problems.append( + f"{ref.label()} is bound now; it no longer awaits " + f"{entry.awaiting}." + ) + continue + if by_concept and ref not in bound: + problems.append(f"{ref.label()} is concept-encoded but not bound.") + elif by_concept and ref not in runs: + problems.append( + f"{ref.label()} is concept-encoded, but its binding reads a " + f"concept {self.content_basis.value} frames drop." + ) + if not by_concept and ref in runs: + problems.append( + f"{ref.label()} is bound by the mapping but closed as " + f"{entry.closure_class.value}" + + (f" ({entry.encoded_by.value})" if entry.encoded_by else "") + + "." + ) + return tuple(problems) + + def _runs(self, binding: object) -> bool: + """Whether a mapping binding runs on this closure's frames.""" + if self.content_basis is not ContentBasis.TRANSPORT: + return True + return not any( + concept(concept_id).transport is TransportRule.DROP + for concept_id in binding.concepts + ) + + # --- Serialization ----------------------------------------------------- + + def to_dict(self) -> dict[str, object]: + """A JSON-ready form that :meth:`from_dict` reads back unchanged.""" + return { + "format": INPUT_CLOSURE_FORMAT, + "country": self.country, + "content_basis": self.content_basis.value, + "mapping": { + "engine": self.mapping_engine, + "engine_version": self.mapping_engine_version, + }, + "input_surface": { + "rulespec_commit": self.surface_rulespec_commit, + "engine_commit": self.surface_engine_commit, + }, + "rulespec_paths": dict(self.modules), + "engine_optional_evidence": self.engine_optional_evidence, + "undetermined_judgments": self.undetermined.to_dict(), + "knobs": [knob.to_dict() for knob in self.knobs], + "entries": [entry.to_dict() for entry in self.entries], + } + + @classmethod + def from_dict(cls, data: Mapping[str, object]) -> InputClosure: + """The closure ``data`` describes, validated like any other. + + Unknown top-level fields are allowed (a country pack's resource can + carry review notes beside the closure); every field the closure + reads is checked. + + Raises: + ValueError: If the format is not :data:`INPUT_CLOSURE_FORMAT`, or + a field is missing or malformed. + """ + with _parsing("input closure"): + if not isinstance(data, Mapping): + raise ValueError("An input closure must be an object.") + if data.get("format") != INPUT_CLOSURE_FORMAT: + raise ValueError( + f"An input closure must declare format {INPUT_CLOSURE_FORMAT!r}." + ) + needed = ( + "country", + "content_basis", + "mapping", + "input_surface", + "rulespec_paths", + "engine_optional_evidence", + "undetermined_judgments", + "knobs", + "entries", + ) + missing = [name for name in needed if name not in data] + if missing: + raise ValueError(f"An input closure lacks {missing}.") + mapping = _record_fields( + data["mapping"], + "closure mapping", + required=("engine", "engine_version"), + ) + surface = _record_fields( + data["input_surface"], + "closure input surface", + required=("rulespec_commit", "engine_commit"), + optional=("fixture",), + ) + _text_fields(mapping, "closure mapping", ("engine", "engine_version")) + _text_fields( + surface, "closure input surface", ("rulespec_commit", "engine_commit") + ) + modules = data["rulespec_paths"] + if not isinstance(modules, Mapping) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in modules.items() + ): + raise ValueError( + "A closure's 'rulespec_paths' maps module paths to SHA-256 text." + ) + policy = _record_fields( + data["undetermined_judgments"], + "undetermined-judgment policy", + required=("action", "reason", "alternatives", "new_default"), + ) + _text_fields(policy, "undetermined-judgment policy", ("action", "reason")) + if not isinstance(policy["alternatives"], list | tuple): + raise ValueError("A policy's 'alternatives' must be a list.") + for name in ("knobs", "entries"): + if not isinstance(data[name], list | tuple): + raise ValueError(f"A closure's {name!r} must be a list.") + evidence = data["engine_optional_evidence"] + if not isinstance(evidence, str): + raise ValueError("A closure's engine evidence must be text.") + for name in ("country", "content_basis"): + if not isinstance(data[name], str): + raise ValueError(f"A closure's {name!r} must be text.") + return cls( + country=data["country"], + content_basis=ContentBasis(data["content_basis"]), + mapping_engine=mapping["engine"], + mapping_engine_version=mapping["engine_version"], + surface_rulespec_commit=surface["rulespec_commit"], + surface_engine_commit=surface["engine_commit"], + modules=modules, + engine_optional_evidence=evidence, + undetermined=UndeterminedPolicy(**policy), + knobs=tuple(Knob.from_dict(knob) for knob in data["knobs"]), + entries=tuple( + ClosureEntry.from_dict(entry) for entry in data["entries"] + ), + ) + + +def apply_defaults( + table: pd.DataFrame, defaults: Mapping[str, bool | int | float] +) -> pd.DataFrame: + """``table`` with one constant column per default. + + Raises: + ValueError: If ``table`` already carries a defaulted input: an input + closed as defaulted is never also encoded. + """ + + clash = sorted(set(defaults) & set(table.columns)) + if clash: + raise ValueError(f"Defaulted inputs {clash} are already encoded.") + out = table.copy() + for name, value in defaults.items(): + dtype = bool if isinstance(value, bool) else type(value) + out[name] = np.full(len(table), value, dtype=dtype) + return out + + +def resolve_judgments(codes: np.ndarray, action: UndeterminedAction) -> np.ndarray: + """Boolean entitlement from engine judgment codes. + + ``1`` holds and ``-1`` does not; ``0`` (undetermined) follows ``action``. + + Raises: + ValueError: If a code is not ``-1``, ``0`` or ``1``, or the action + refuses an undetermined judgment that occurs. + """ + + action = UndeterminedAction(action) + values = np.asarray(codes) + if values.size == 0: + return np.zeros(values.shape, dtype=bool) + if values.dtype.kind not in "iu" or not np.isin(values, (-1, 0, 1)).all(): + raise ValueError("Judgment codes are -1, 0 or 1.") + undetermined = values == 0 + if action is UndeterminedAction.REFUSE and undetermined.any(): + raise ValueError( + f"{int(undetermined.sum())} judgment(s) are undetermined, and the " + "closure refuses them." + ) + return np.where(undetermined, action is UndeterminedAction.ELIGIBLE, values == 1) diff --git a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py new file mode 100644 index 0000000000..d6876768a0 --- /dev/null +++ b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py @@ -0,0 +1,678 @@ +"""Benefit units built from concept relationship pointers. + +A transported concept frame (:mod:`microcosm.frame.concepts`) has persons and +households only. A target country's rules assess many programs on a smaller +unit: a New Zealand benefit is assessed on a person, their partner and their +dependent children. This module builds that unit from the concept pointers +alone (``partner_person_id``, ``parent_1_person_id``, ``parent_2_person_id`` +and the household's ``reference_person_id``), so no donor-country roster +column reaches the target engine. :mod:`microcosm.frame.units` is the +precedent for building unit structure in this package; it builds the US +units from raw CPS roster columns instead. + +The rule +-------- +Who counts as a dependent child is law, so it is a declared +:class:`BenefitUnitRule`, never a constant here. Its values come from the +country's RuleSpec where the RuleSpec encodes them and otherwise from the +country's spec data, with their citations. Given the rule: + +1. A person is a *candidate child* when their age is at most the rule's + ``max_age``, they have no co-resident partner, no co-resident person names + them as a parent, and, when the rule declares a financial-independence + test, they do not meet it. A partnered person and a parent are never + dependent children, which keeps partners together and parents with their + own children. +2. A candidate child with a co-resident parent is a *dependent child* of + that parent's unit. A candidate child with none who is not the household's + reference person is placed by the rule's ``unparented_child`` policy: in + the unit that contains the household's reference person (the caregiver + the pointers can name), in a unit of their own, or refused. A reference + person is never their own caregiver, so a candidate child who is the + reference person and has no co-resident parent heads their own unit. +3. Every person who is not a dependent child heads a unit with their + co-resident partner, if any. These are the units' adults: every unit has + one or two. + +A child whose two co-resident parents are in different units (two parents who +are not partners of each other) is ambiguous; the rule's ``split_parents`` +policy refuses the frame or places the child with ``parent_1_person_id``. +Pointers that leave the household, dangle, or break any other concept-frame +contract are refused before any unit is built +(:func:`~microcosm.frame.concepts.validate_concept_tables`), so units always +nest in households. + +Identity +-------- +A unit's id is its head's ``person_id``: unique, deterministic, and +independent of row order, so rebuilding from the same records reproduces the +same ids. A unit's head is one of its adults: the household's reference person +when they are one of them, otherwise the older adult of a couple, ties broken +by the smaller ``person_id`` (a reference person who is a dependent child is +never a head). +The unit table is sorted by id, as a :class:`~microcosm.frame.bundle.Frame` +group table must be, and carries the head (``_head_person_id``) and +the household the unit nests in (``_household_id``); consumers read +those columns rather than rely on the id's value. Units carry no weight of +their own: a unit takes its household's weight through its members +(:meth:`~microcosm.frame.bundle.Frame.resolve_weights`). +""" + +from __future__ import annotations + +import math +import re +from collections.abc import Mapping +from dataclasses import dataclass +from enum import StrEnum + +import numpy as np +import pandas as pd + +from microcosm.frame.concept_mapping import GroupMembership, UnitRole +from microcosm.frame.concepts import ( + HOUSEHOLD_ID_COLUMN, + PERSON_HOUSEHOLD_ID_COLUMN, + PERSON_ID_COLUMN, + _parsing, + _record_fields, + validate_concept_tables, +) + +__all__ = [ + "BENEFIT_UNIT_RULE_FORMAT", + "BenefitUnitRule", + "DependentChildRule", + "FinancialIndependenceTest", + "SplitParentsPolicy", + "UnparentedChildPlacement", + "benefit_unit_attributes", + "benefit_unit_membership", + "benefit_unit_roles", + "build_benefit_units", + "units_per_household", +] + +#: The serialized rule format :meth:`BenefitUnitRule.to_dict` writes. +BENEFIT_UNIT_RULE_FORMAT = "microcosm.benefit_unit_rule.v1" + +_PERSON_ID = PERSON_ID_COLUMN +_PERSON_HOUSEHOLD_ID = PERSON_HOUSEHOLD_ID_COLUMN +_HOUSEHOLD_ID = HOUSEHOLD_ID_COLUMN +_ENTITY_NAME = re.compile(r"^[a-z][a-z0-9_]*$") +_POINTERS = ("partner_person_id", "parent_1_person_id", "parent_2_person_id") + + +class UnparentedChildPlacement(StrEnum): + """Where a candidate child with no co-resident parent goes. + + A candidate child who is the household's reference person heads their + own unit whatever the placement: they cannot be their own caregiver. + """ + + #: Into the unit that contains the household's reference person. + REFERENCE_PERSON_UNIT = "reference_person_unit" + #: Into a unit of their own: they are not a dependent child. + OWN_UNIT = "own_unit" + #: Nowhere: the frame is refused. + REFUSE = "refuse" + + +class SplitParentsPolicy(StrEnum): + """What happens to a child whose two co-resident parents are not partners.""" + + #: The frame is refused: the child's unit is ambiguous. + REFUSE = "refuse" + #: The child joins the unit of ``parent_1_person_id``. + FIRST_PARENT_UNIT = "first_parent_unit" + + +@dataclass(frozen=True, kw_only=True) +class FinancialIndependenceTest: + """When a candidate child counts as financially independent. + + A person at least :attr:`min_age` old who usually works at least + :attr:`min_usual_weekly_hours` a week is not a dependent child. Only the + employment arm of a statutory test can be read from concepts; receipt of + a benefit or allowance is model state that depends on the units, so it + cannot decide them. + """ + + min_age: int + min_usual_weekly_hours: float + + def __post_init__(self) -> None: + if isinstance(self.min_age, bool) or not isinstance(self.min_age, int): + raise ValueError("A financial-independence age must be an integer.") + hours = self.min_usual_weekly_hours + if isinstance(hours, bool) or not isinstance(hours, int | float): + raise ValueError("Financial-independence hours must be a number.") + hours = float(hours) + if not math.isfinite(hours) or not 0.0 < hours <= 168.0: + raise ValueError( + f"Financial-independence hours must lie on (0, 168], got {hours}." + ) + if self.min_age < 0: + raise ValueError("A financial-independence age cannot be negative.") + object.__setattr__(self, "min_usual_weekly_hours", hours) + + +@dataclass(frozen=True, kw_only=True) +class DependentChildRule: + """Who can be a dependent child. + + Attributes: + max_age: The oldest age, in completed years, a dependent child can be. + financial_independence: The test that takes a candidate out of + dependency, or ``None`` when the rule declares none. + """ + + max_age: int + financial_independence: FinancialIndependenceTest | None + + def __post_init__(self) -> None: + if isinstance(self.max_age, bool) or not isinstance(self.max_age, int): + raise ValueError("A dependent child's maximum age must be an integer.") + if not 0 <= self.max_age <= 130: + raise ValueError(f"max_age must lie on [0, 130], got {self.max_age}.") + test = self.financial_independence + if test is not None and not isinstance(test, FinancialIndependenceTest): + raise ValueError("A financial-independence test must be declared as one.") + if test is not None and test.min_age > self.max_age: + raise ValueError( + "A financial-independence age above max_age can never apply." + ) + + +@dataclass(frozen=True, kw_only=True) +class BenefitUnitRule: + """How a country groups persons into benefit units. + + Attributes: + entity: The frame group entity the units become (``family``). Its + table carries ``_id``, ``_household_id`` and + ``_head_person_id``; persons carry + ``person__id``. + dependent_child: Who can be a dependent child. + unparented_child: Where a candidate child with no co-resident parent + goes. + split_parents: What happens to a child whose co-resident parents are + in different units. + provenance: Where each value comes from (statute, RuleSpec rule or + spec data, with citations). Carried verbatim for review; it does + not change any unit. + """ + + entity: str + dependent_child: DependentChildRule + unparented_child: UnparentedChildPlacement + split_parents: SplitParentsPolicy + provenance: Mapping[str, str] + + def __post_init__(self) -> None: + if not isinstance(self.entity, str) or not _ENTITY_NAME.match(self.entity): + raise ValueError(f"A unit entity must be snake_case, got {self.entity!r}.") + if self.entity in ("person", "household"): + raise ValueError(f"Units cannot be named {self.entity!r}.") + if not isinstance(self.dependent_child, DependentChildRule): + raise ValueError("A unit rule needs a DependentChildRule.") + object.__setattr__( + self, "unparented_child", UnparentedChildPlacement(self.unparented_child) + ) + object.__setattr__( + self, "split_parents", SplitParentsPolicy(self.split_parents) + ) + if not isinstance(self.provenance, Mapping) or not all( + isinstance(key, str) and isinstance(text, str) and key and text + for key, text in self.provenance.items() + ): + raise ValueError("A unit rule's provenance maps text to non-empty text.") + object.__setattr__(self, "provenance", dict(self.provenance)) + + def __hash__(self) -> int: + return hash( + ( + self.entity, + self.dependent_child, + self.unparented_child, + self.split_parents, + tuple(sorted(self.provenance.items())), + ) + ) + + @property + def id_column(self) -> str: + """The unit table's id column.""" + return f"{self.entity}_id" + + @property + def household_column(self) -> str: + """The unit table's column naming the household the unit nests in.""" + return f"{self.entity}_household_id" + + @property + def head_column(self) -> str: + """The unit table's column naming the unit's head.""" + return f"{self.entity}_head_person_id" + + @property + def membership_column(self) -> str: + """The person table's column naming each person's unit.""" + return f"person_{self.entity}_id" + + def to_dict(self) -> dict[str, object]: + """A JSON-ready form that :meth:`from_dict` reads back unchanged.""" + test = self.dependent_child.financial_independence + return { + "format": BENEFIT_UNIT_RULE_FORMAT, + "entity": self.entity, + "dependent_child": { + "max_age": self.dependent_child.max_age, + "financial_independence": None + if test is None + else { + "min_age": test.min_age, + "min_usual_weekly_hours": test.min_usual_weekly_hours, + }, + }, + "unparented_child": self.unparented_child.value, + "split_parents": self.split_parents.value, + "provenance": dict(self.provenance), + } + + @classmethod + def from_dict(cls, data: Mapping[str, object]) -> BenefitUnitRule: + """The rule ``data`` describes (see :meth:`to_dict`). + + Raises: + ValueError: If the format is not :data:`BENEFIT_UNIT_RULE_FORMAT`, + or a field is missing, unexpected or malformed. + """ + with _parsing("benefit unit rule"): + fields = _record_fields( + data, + "benefit unit rule", + required=( + "format", + "entity", + "dependent_child", + "unparented_child", + "split_parents", + "provenance", + ), + ) + if fields.pop("format") != BENEFIT_UNIT_RULE_FORMAT: + raise ValueError( + f"A benefit unit rule must declare format " + f"{BENEFIT_UNIT_RULE_FORMAT!r}." + ) + child = _record_fields( + fields["dependent_child"], + "dependent child rule", + required=("max_age", "financial_independence"), + ) + raw_test = child["financial_independence"] + test = None + if raw_test is not None: + test_fields = _record_fields( + raw_test, + "financial independence test", + required=("min_age", "min_usual_weekly_hours"), + ) + test = FinancialIndependenceTest(**test_fields) + fields["dependent_child"] = DependentChildRule( + max_age=child["max_age"], financial_independence=test + ) + for name in ("entity", "unparented_child", "split_parents"): + if not isinstance(fields[name], str): + raise ValueError(f"A benefit unit rule's {name!r} must be text.") + if not isinstance(fields["provenance"], Mapping): + raise ValueError( + "A benefit unit rule's 'provenance' must be an object." + ) + return cls(**fields) + + +def build_benefit_units( + person: pd.DataFrame, + household: pd.DataFrame, + rule: BenefitUnitRule, +) -> tuple[pd.DataFrame, pd.Series]: + """Group a concept frame's persons into benefit units. + + Args: + person: The concept frame's person table. It needs ``person_id``, + ``person_household_id``, ``age`` and the three person pointers, + plus ``usual_weekly_hours`` when the rule declares a + financial-independence test. + household: The concept frame's household table, with + ``household_id`` and ``reference_person_id``. + rule: The country's benefit-unit rule. + + Returns: + The unit table, one row per unit sorted by id, with ``_id``, + ``_household_id`` and ``_head_person_id`` (all + ``int64``); and each person's unit id (``int64``), aligned to + ``person``'s rows and index and named ``person__id``. + + Raises: + ValueError: If a needed column is missing, the frame breaks a + concept-frame contract (pointers that dangle or leave the + household, asymmetric partners, parent cycles, a reference person + outside their household, bad ages or hours), or the rule refuses + a child's placement. + """ + + rows = _Pointers.read(person, household, rule) + dependent = _dependents(rows, rule) + head = _heads(rows, dependent, rule) + ids = rows.ids + head_rows = np.unique(head) + order = np.argsort(ids[head_rows], kind="stable") + head_rows = head_rows[order] + family = pd.DataFrame( + { + rule.id_column: ids[head_rows], + rule.household_column: rows.household_ids[head_rows], + rule.head_column: ids[head_rows], + } + ) + membership = pd.Series( + ids[head], index=person.index, name=rule.membership_column, dtype=np.int64 + ) + return family, membership + + +def benefit_unit_roles( + person: pd.DataFrame, + family: pd.DataFrame, + membership: pd.Series, + rule: BenefitUnitRule, +) -> pd.Series: + """Each person's role in their unit, read back from the built units. + + A unit is its head, the head's co-resident partner when the partner shares + the unit, and dependent children; so the roles follow from the unit table + and the partner pointers without re-running the rule. + + Returns: + A string series aligned to ``person``, with values from + :class:`~microcosm.frame.concept_mapping.UnitRole`. + """ + + ids = person[_PERSON_ID].to_numpy(dtype=np.int64) + units = _unit_rows(family, membership, rule) + head_of = family[rule.head_column].to_numpy(dtype=np.int64)[units] + is_head = ids == head_of + present = person["partner_person_id"].notna().to_numpy() + partner = np.full(len(ids), -1, dtype=np.int64) + partner[present] = person["partner_person_id"][present].to_numpy(dtype=np.int64) + head_partner = pd.Series(partner, index=ids).reindex(head_of).to_numpy() + is_partner = ~is_head & (head_partner == ids) + roles = np.where( + is_head, + UnitRole.HEAD.value, + np.where( + is_partner, + UnitRole.PARTNER.value, + UnitRole.DEPENDENT_CHILD.value, + ), + ) + return pd.Series( + roles, index=person.index, name=f"{rule.entity}_role", dtype=object + ) + + +def benefit_unit_membership( + person: pd.DataFrame, + family: pd.DataFrame, + membership: pd.Series, + rule: BenefitUnitRule, +) -> GroupMembership: + """The built units as :meth:`ConceptMapping.encode_groups` reads them.""" + + return GroupMembership( + entity=rule.entity, + units=family, + person_unit=membership, + person_role=benefit_unit_roles(person, family, membership, rule), + ) + + +def benefit_unit_attributes( + person: pd.DataFrame, + family: pd.DataFrame, + membership: pd.Series, + rule: BenefitUnitRule, +) -> pd.DataFrame: + """Composition attributes of each built unit. + + Returns: + A table aligned to ``family``'s rows with ``_id`` and: + + - ``n_members``, ``n_adults`` and ``n_dependent_children`` (``int64``); + the adults are the head and a partner who shares the unit; + - ``is_couple``: the unit has two adults; + - ``is_sole_parent``: one adult and at least one dependent child; + - ``family_type``: ``single``, ``couple``, ``sole_parent`` or + ``couple_with_children``; + - ``youngest_dependent_child_age``: nullable ``Int64``, null when the + unit has no dependent child. + """ + + roles = benefit_unit_roles(person, family, membership, rule).to_numpy() + unit_rows = _unit_rows(family, membership, rule) + n_units = len(family) + is_adult = roles != UnitRole.DEPENDENT_CHILD.value + n_members = np.bincount(unit_rows, minlength=n_units).astype(np.int64) + n_adults = np.bincount(unit_rows[is_adult], minlength=n_units).astype(np.int64) + n_children = n_members - n_adults + ages = person["age"].to_numpy(dtype=np.float64) + youngest = np.full(n_units, np.inf) + child_rows = np.flatnonzero(~is_adult) + np.minimum.at(youngest, unit_rows[child_rows], ages[child_rows]) + youngest_age = pd.array( + [None if math.isinf(value) else int(value) for value in youngest], + dtype="Int64", + ) + is_couple = n_adults == 2 + with_children = n_children > 0 + family_type = np.where( + is_couple, + np.where(with_children, "couple_with_children", "couple"), + np.where(with_children, "sole_parent", "single"), + ) + return pd.DataFrame( + { + rule.id_column: family[rule.id_column].to_numpy(dtype=np.int64), + "n_members": n_members, + "n_adults": n_adults, + "n_dependent_children": n_children, + "is_couple": is_couple, + "is_sole_parent": ~is_couple & with_children, + "family_type": family_type.astype(object), + "youngest_dependent_child_age": youngest_age, + } + ) + + +def units_per_household( + household: pd.DataFrame, family: pd.DataFrame, rule: BenefitUnitRule +) -> pd.Series: + """How many units each household holds, aligned to ``household``'s rows.""" + + counts = family[rule.household_column].value_counts() + return pd.Series( + counts.reindex(household[_HOUSEHOLD_ID].to_numpy(), fill_value=0) + .to_numpy() + .astype(np.int64), + index=household.index, + name=f"n_{rule.entity}_units", + ) + + +# --- Construction helpers -------------------------------------------------- + + +@dataclass(frozen=True, eq=False) +class _Pointers: + """Pointer targets resolved to row positions (-1 when absent).""" + + ids: np.ndarray + household_ids: np.ndarray + age: np.ndarray + hours: np.ndarray | None + partner: np.ndarray + parent_1: np.ndarray + parent_2: np.ndarray + reference: np.ndarray + + @classmethod + def read( + cls, + person: pd.DataFrame, + household: pd.DataFrame, + rule: BenefitUnitRule, + ) -> _Pointers: + test = rule.dependent_child.financial_independence + person_columns = [_PERSON_ID, _PERSON_HOUSEHOLD_ID, "age", *_POINTERS] + if test is not None: + person_columns.append("usual_weekly_hours") + missing = [name for name in person_columns if name not in person.columns] + missing += [ + f"household.{name}" + for name in (_HOUSEHOLD_ID, "reference_person_id") + if name not in household.columns + ] + if missing: + raise ValueError(f"Building benefit units needs columns {missing}.") + subset = { + "person": person.loc[:, person_columns], + "household": household.loc[:, [_HOUSEHOLD_ID, "reference_person_id"]], + } + violations = validate_concept_tables(subset) + if violations: + detail = "; ".join( + f"{item.entity}.{item.column} {item.code}: {item.message}" + + (f" ({item.rows} rows)" if item.rows else "") + for item in violations + ) + raise ValueError(f"Benefit units refuse an invalid concept frame: {detail}") + ids = person[_PERSON_ID].to_numpy(dtype=np.int64) + index = pd.Index(ids) + + def rows(values: pd.Series) -> np.ndarray: + present = values.notna().to_numpy() + found = np.full(len(values), -1, dtype=np.int64) + if present.any(): + found[present] = index.get_indexer( + values[present].to_numpy(dtype=np.int64) + ) + return found + + household_rows = pd.Index( + household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64) + ).get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64)) + reference_by_household = rows(household["reference_person_id"]) + return cls( + ids=ids, + household_ids=person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64), + age=person["age"].to_numpy(dtype=np.float64), + hours=None + if test is None + else person["usual_weekly_hours"].to_numpy(dtype=np.float64), + partner=rows(person["partner_person_id"]), + parent_1=rows(person["parent_1_person_id"]), + parent_2=rows(person["parent_2_person_id"]), + reference=reference_by_household[household_rows], + ) + + +def _dependents(rows: _Pointers, rule: BenefitUnitRule) -> np.ndarray: + """Which persons are dependent children under ``rule``.""" + + n = len(rows.ids) + named = np.concatenate( + [rows.parent_1[rows.parent_1 >= 0], rows.parent_2[rows.parent_2 >= 0]] + ) + is_parent = np.bincount(named, minlength=n) > 0 + child_rule = rule.dependent_child + candidate = (rows.age <= child_rule.max_age) & (rows.partner < 0) & ~is_parent + test = child_rule.financial_independence + if test is not None: + assert rows.hours is not None + candidate &= ~( + (rows.age >= test.min_age) & (rows.hours >= test.min_usual_weekly_hours) + ) + has_parent = rows.parent_1 >= 0 + is_reference = rows.reference == np.arange(n) + # A candidate with no co-resident parent who is the reference person + # heads their own unit; every other unparented candidate needs placing. + unparented = candidate & ~has_parent & ~is_reference + placement = rule.unparented_child + if placement is UnparentedChildPlacement.REFUSE and unparented.any(): + raise ValueError( + f"The unit rule refuses {int(unparented.sum())} candidate " + "child(ren) with no co-resident parent." + ) + with_parent = candidate & has_parent + if placement is UnparentedChildPlacement.REFERENCE_PERSON_UNIT: + return with_parent | unparented + return with_parent + + +def _heads(rows: _Pointers, dependent: np.ndarray, rule: BenefitUnitRule) -> np.ndarray: + """Each person's unit head, as a person row.""" + + n = len(rows.ids) + own = np.arange(n) + head = np.full(n, -1, dtype=np.int64) + adult = ~dependent + partner = rows.partner + is_reference = rows.reference == own + # A partnered person is never a candidate child, so both partners are + # adults here; validation guarantees the pointers are symmetric. + paired = adult & (partner >= 0) + other = np.where(paired, partner, own) + mine_first = ( + is_reference + | (~is_reference[other] & (rows.age > rows.age[other])) + | ( + ~is_reference[other] + & (rows.age == rows.age[other]) + & (rows.ids <= rows.ids[other]) + ) + ) + head[adult] = np.where(mine_first, own, other)[adult] + + # A parent is never a candidate child, so every parent's head is known. + with_parent = dependent & (rows.parent_1 >= 0) + first = head[rows.parent_1[with_parent]] + second_rows = rows.parent_2[with_parent] + second = np.where(second_rows >= 0, head[np.maximum(second_rows, 0)], first) + split = second != first + if split.any() and rule.split_parents is SplitParentsPolicy.REFUSE: + raise ValueError( + f"{int(split.sum())} dependent child(ren) have co-resident parents " + "in different units; the unit rule refuses the ambiguity." + ) + head[with_parent] = first + + # Unparented dependent children join the unit of the household's + # reference person, who is an adult or a dependent child with a parent, + # so their head is already known. + unparented = dependent & (rows.parent_1 < 0) + head[unparented] = head[rows.reference[unparented]] + if (head < 0).any(): # pragma: no cover - construction guarantees it + raise AssertionError("A person was left without a benefit unit.") + return head + + +def _unit_rows( + family: pd.DataFrame, membership: pd.Series, rule: BenefitUnitRule +) -> np.ndarray: + unit_ids = family[rule.id_column].to_numpy(dtype=np.int64) + rows = pd.Index(unit_ids).get_indexer(membership.to_numpy(dtype=np.int64)) + if (rows < 0).any(): + raise ValueError("A person's unit is missing from the unit table.") + return rows diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py new file mode 100644 index 0000000000..5118e2e67b --- /dev/null +++ b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py @@ -0,0 +1,1002 @@ +"""Group encoding: concept and state bindings executed on built units. + +``ConceptMapping.encode_groups`` executes the group rules a mapping declares, +on units a unit-construction step built from the pointers. Its invariants run +as properties over generated concept frames and unit rules: + +- ``household_value`` is constant within a household; +- ``allocate_to_reference_unit`` puts each household amount on exactly the + unit that holds the reference person, and conserves it; the per-adult + alternative conserves it too; +- ``sum_over_members`` conserves member totals; ``any_member`` and + ``reference_member`` read the members and the head; +- encoding is deterministic and row-aligned to the unit table. + +A row-by-row reference written from each rule's documented meaning checks the +vectorized executor (a differential test), and unit-composition flags are +checked against :func:`benefit_unit_attributes`, a second implementation of +the same composition. +""" + +import json + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from microcosm.frame.adapters.axiom import axiom_concept_mapping +from microcosm.frame.concept_mapping import ( + Allocation, + ConceptMapping, + GroupKnobs, + GroupMembership, + GroupRule, + Identity, + InputBinding, + InputDeclaration, + Predicate, + RelationshipRole, + Role, + Scale, + ScaledSum, + StateBinding, + StateTest, + TakeUpThreshold, + UnitComposition, + UnitFeature, + UnitRole, + transform_from_dict, + transform_to_dict, +) +from microcosm.frame.concepts import CONCEPTS, AlignmentRelation +from microcosm.frame.unit_construction import ( + BenefitUnitRule, + DependentChildRule, + benefit_unit_attributes, + benefit_unit_membership, + build_benefit_units, +) +from test_support.microcosm_frame.concept_frames import concept_frames +from test_support.microcosm_frame.unit_rules import unit_rules + +PROPERTY = settings(max_examples=120, deadline=None) +WEEK = 1 / 52 +UNIT_CONCEPTS = UnitComposition(feature=UnitFeature.SOLE_PARENT).concepts + + +def _binding(engine_input: str, concepts, transform, rule, **change) -> InputBinding: + return InputBinding( + engine_input=engine_input, + engine_entity=change.pop("engine_entity", "Family"), + concepts=tuple(concepts), + transform=transform, + relation=AlignmentRelation.APPROXIMATE, + note="evidence", + group_rule=rule, + **change, + ) + + +BINDINGS = ( + _binding( + "earnings", + ("fact:person.employment_income", "fact:person.interest_income"), + ScaledSum(factor=WEEK), + GroupRule.SUM_OVER_MEMBERS, + ), + _binding( + "weeks", + ("fact:person.weeks_worked",), + Identity(), + GroupRule.SUM_OVER_MEMBERS, + ), + _binding( + "any_disability", + ("fact:person.has_disability",), + Identity(), + GroupRule.ANY_MEMBER, + ), + _binding("head_age", ("fact:person.age",), Identity(), GroupRule.REFERENCE_MEMBER), + _binding( + "head_partnered", + ("fact:person.partner_person_id",), + RelationshipRole(role=Role.HAS_PARTNER), + GroupRule.REFERENCE_MEMBER, + ), + _binding( + "owner", + ("fact:household.tenure",), + Predicate( + clauses=( + ("fact:household.tenure", ("owned_outright", "owned_with_mortgage")), + ) + ), + GroupRule.HOUSEHOLD_VALUE, + ), + _binding( + "rent", + ("fact:household.rent",), + Scale(factor=WEEK), + GroupRule.ALLOCATE_TO_REFERENCE_UNIT, + ), + _binding( + "mortgage", + ("fact:household.mortgage_interest", "fact:household.mortgage_principal"), + ScaledSum(factor=WEEK), + GroupRule.ALLOCATE_TO_REFERENCE_UNIT, + ), + *( + _binding( + f"unit_{feature.value}", + UNIT_CONCEPTS, + UnitComposition(feature=feature), + GroupRule.REFERENCE_MEMBER, + ) + for feature in UnitFeature + ), + _binding( + "child_age", + ("fact:person.age",), + Identity(), + GroupRule.PERSON_ROLE, + engine_entity="Child", + ), + _binding( + "tax_unit_income", + ("fact:person.employment_income",), + Identity(), + GroupRule.SUM_OVER_MEMBERS, + engine_entity="TaxUnit", + ), +) + + +def _mapping(bindings=BINDINGS) -> ConceptMapping: + read = {concept_id for binding in bindings for concept_id in binding.reads} + return ConceptMapping( + engine="test", + engine_version="0", + entity_correspondence={"person": "Person", "household": "Household"}, + input_declaration=InputDeclaration.ENGINE_TYPED, + bindings=tuple(bindings), + unmapped={ + item.id: "not in this test" for item in CONCEPTS if item.id not in read + }, + ) + + +MAPPING = _mapping() +STATE = ( + StateBinding( + engine_input="non_recipient", + engine_entity="Family", + columns=("person.receives_a", "person.receives_b"), + test=StateTest.ANY_TRUE, + group_rule=GroupRule.ANY_MEMBER, + negate=True, + note="evidence", + ), + StateBinding( + engine_input="head_receives_a", + engine_entity="Family", + columns=("person.receives_a",), + test=StateTest.ANY_TRUE, + group_rule=GroupRule.REFERENCE_MEMBER, + note="evidence", + ), + StateBinding( + engine_input="in_area_1", + engine_entity="Family", + columns=("household.area",), + test=StateTest.EQUALS, + value=1, + group_rule=GroupRule.HOUSEHOLD_VALUE, + note="evidence", + ), +) + + +@st.composite +def unit_frames(draw): + """A concept frame with model state, its units, and its membership.""" + + tables = draw(concept_frames(max_households=5, max_members=6)) + rule = draw(unit_rules()) + person = tables["person"].copy() + household = tables["household"].copy() + for column in ("receives_a", "receives_b"): + person[column] = np.asarray( + draw(st.lists(st.booleans(), min_size=len(person), max_size=len(person))), + dtype=bool, + ) + household["area"] = np.asarray( + draw( + st.lists( + st.integers(1, 4), min_size=len(household), max_size=len(household) + ) + ), + dtype=np.int64, + ) + household["area_alt"] = np.asarray( + draw( + st.lists( + st.integers(1, 4), min_size=len(household), max_size=len(household) + ) + ), + dtype=np.int64, + ) + family, membership = build_benefit_units(person, household, rule) + group = benefit_unit_membership(person, family, membership, rule) + return {"person": person, "household": household}, rule, family, membership, group + + +def _encode(tables, group, **kwargs): + return MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=STATE, **kwargs + ) + + +# --- A row-by-row reference, straight from each rule's documented meaning ---- + + +def _reference(tables, rule, family, membership, *, allocation="reference_unit"): + person = tables["person"].to_dict("records") + households = { + row["household_id"]: row for row in tables["household"].to_dict("records") + } + unit_of = dict(zip([row["person_id"] for row in person], membership, strict=True)) + # Roles from the unit table and the partner pointers alone, not from + # benefit_unit_roles: the head is the unit's head, the head's partner in + # the same unit is the partner, and every other member is a child. + head_of_unit = dict( + zip(family[rule.id_column], family[rule.head_column], strict=True) + ) + partner_of = { + row["person_id"]: None + if pd.isna(row["partner_person_id"]) + else row["partner_person_id"] + for row in person + } + roles = {} + for row in person: + pid = row["person_id"] + head = head_of_unit[unit_of[pid]] + if pid == head: + roles[pid] = UnitRole.HEAD.value + elif partner_of[head] == pid: + roles[pid] = UnitRole.PARTNER.value + else: + roles[pid] = UnitRole.DEPENDENT_CHILD.value + out = {name: [] for name in ("earnings", "weeks", "any_disability", "head_age")} + out |= {"head_partnered": [], "owner": [], "rent": [], "mortgage": []} + out |= {"non_recipient": [], "head_receives_a": [], "in_area_1": []} + out |= {f"unit_{feature.value}": [] for feature in UnitFeature} + for unit in family.to_dict("records"): + members = [ + row for row in person if unit_of[row["person_id"]] == unit[rule.id_column] + ] + head = next( + row for row in members if row["person_id"] == unit[rule.head_column] + ) + home = households[unit[rule.household_column]] + out["earnings"].append( + sum( + (row["employment_income"] + row["interest_income"]) * WEEK + for row in members + ) + ) + out["weeks"].append(sum(row["weeks_worked"] for row in members)) + out["any_disability"].append(any(row["has_disability"] for row in members)) + out["head_age"].append(head["age"]) + out["head_partnered"].append(not pd.isna(head["partner_person_id"])) + out["owner"].append(home["tenure"] in ("owned_outright", "owned_with_mortgage")) + holds_reference = any( + row["person_id"] == home["reference_person_id"] for row in members + ) + mortgage = (home["mortgage_interest"] + home["mortgage_principal"]) * WEEK + out["mortgage"].append(mortgage if holds_reference else 0.0) + if allocation == "reference_unit": + out["rent"].append(home["rent"] * WEEK if holds_reference else 0.0) + else: + adults = { + key: sum( + 1 + for row in person + if unit_of[row["person_id"]] == key + and roles[row["person_id"]] != UnitRole.DEPENDENT_CHILD.value + ) + for key in set(unit_of.values()) + } + in_home = [ + key + for key, household_id in zip( + family[rule.id_column], family[rule.household_column], strict=True + ) + if household_id == unit[rule.household_column] + ] + share = adults[unit[rule.id_column]] / sum(adults[key] for key in in_home) + out["rent"].append(home["rent"] * WEEK * share) + out["non_recipient"].append( + not any(row["receives_a"] or row["receives_b"] for row in members) + ) + out["head_receives_a"].append(bool(head["receives_a"])) + out["in_area_1"].append(home["area"] == 1) + partners = sum( + roles[row["person_id"]] == UnitRole.PARTNER.value for row in members + ) + children = sum( + roles[row["person_id"]] == UnitRole.DEPENDENT_CHILD.value for row in members + ) + out["unit_has_partner"].append(partners > 0) + out["unit_has_dependent_child"].append(children >= 1) + out["unit_two_or_more_dependent_children"].append(children >= 2) + out["unit_sole_parent"].append(partners == 0 and children >= 1) + return out + + +class TestExecution: + @PROPERTY + @given(case=unit_frames(), mode=st.sampled_from(list(Allocation))) + def test_encode_groups_matches_a_row_by_row_reference(self, case, mode) -> None: + tables, rule, family, membership, group = case + encoded = _encode(tables, group, knobs=GroupKnobs(allocation={"rent": mode})) + table = encoded.tables[rule.entity] + expected = _reference(tables, rule, family, membership, allocation=mode.value) + assert table[rule.id_column].tolist() == family[rule.id_column].tolist() + for name, values in expected.items(): + actual = table[name].to_numpy() + if isinstance(values[0], bool): + assert actual.dtype == bool, name + assert actual.tolist() == values, name + else: + np.testing.assert_allclose( + actual.astype(np.float64), + values, + rtol=1e-12, + atol=1e-9, + err_msg=name, + ) + + @PROPERTY + @given(case=unit_frames()) + def test_household_values_are_constant_within_a_household(self, case) -> None: + tables, rule, family, _, group = case + table = _encode(tables, group).tables[rule.entity] + households = family[rule.household_column].to_numpy() + for name in ("owner", "in_area_1"): + per_household = pd.Series(table[name].to_numpy()).groupby(households) + assert (per_household.nunique() == 1).all(), name + + @PROPERTY + @given(case=unit_frames(), mode=st.sampled_from(list(Allocation))) + def test_allocation_conserves_each_household_amount(self, case, mode) -> None: + tables, rule, family, membership, group = case + table = _encode( + tables, group, knobs=GroupKnobs(allocation={"rent": mode}) + ).tables[rule.entity] + households = family[rule.household_column].to_numpy() + household = tables["household"].set_index("household_id") + for name, amount in ( + ("rent", household["rent"] * WEEK), + ( + "mortgage", + (household["mortgage_interest"] + household["mortgage_principal"]) + * WEEK, + ), + ): + totals = pd.Series(table[name].to_numpy()).groupby(households).sum() + np.testing.assert_allclose( + totals.to_numpy(), amount.reindex(totals.index).to_numpy(), rtol=1e-12 + ) + # reference_unit puts the amount on exactly the reference person's unit + reference_unit = pd.Series( + membership.to_numpy(), index=tables["person"]["person_id"].to_numpy() + ).reindex(household["reference_person_id"].astype("int64").to_numpy()) + holds = family[rule.id_column].isin(reference_unit.to_numpy()).to_numpy() + assert (pd.Series(holds).groupby(households).sum() == 1).all() + assert (table.loc[~holds, "mortgage"] == 0).all() + if mode is Allocation.REFERENCE_UNIT: + assert (table.loc[~holds, "rent"] == 0).all() + + @PROPERTY + @given(case=unit_frames()) + def test_sums_conserve_member_totals(self, case) -> None: + tables, rule, _, _, group = case + table = _encode(tables, group).tables[rule.entity] + person = tables["person"] + np.testing.assert_allclose( + table["earnings"].sum(), + ((person["employment_income"] + person["interest_income"]) * WEEK).sum(), + rtol=1e-9, + atol=1e-6, + ) + assert table["weeks"].dtype == np.int64 + assert table["weeks"].sum() == person["weeks_worked"].sum() + + @PROPERTY + @given(case=unit_frames()) + def test_unit_composition_agrees_with_unit_attributes(self, case) -> None: + tables, rule, family, membership, group = case + table = _encode(tables, group).tables[rule.entity] + attributes = benefit_unit_attributes(tables["person"], family, membership, rule) + children = attributes["n_dependent_children"].to_numpy() + assert table["unit_has_partner"].tolist() == attributes["is_couple"].tolist() + assert table["unit_has_dependent_child"].tolist() == (children >= 1).tolist() + assert ( + table["unit_two_or_more_dependent_children"].tolist() + == (children >= 2).tolist() + ) + assert ( + table["unit_sole_parent"].tolist() == attributes["is_sole_parent"].tolist() + ) + # the head's partner pointer and the unit's partner role agree + assert table["head_partnered"].tolist() == table["unit_has_partner"].tolist() + + @PROPERTY + @given(case=unit_frames(), factor=st.floats(0.0, 3.0, allow_nan=False)) + def test_knobs_zero_scale_and_redirect(self, case, factor) -> None: + tables, rule, _, _, group = case + base = _encode(tables, group).tables[rule.entity] + knobbed = _encode( + tables, + group, + knobs=GroupKnobs( + zeroed=frozenset({"earnings"}), + factors={"rent": factor}, + state_columns={"household.area": "household.area_alt"}, + ), + ).tables[rule.entity] + assert (knobbed["earnings"] == 0).all() + np.testing.assert_allclose(knobbed["rent"], base["rent"] * factor, rtol=1e-12) + alternative = tables["household"].set_index("household_id")["area_alt"] + expected = ( + alternative.reindex(group.units[group.household_column]).to_numpy() == 1 + ) + assert knobbed["in_area_1"].tolist() == expected.tolist() + pd.testing.assert_frame_equal( + knobbed.drop(columns=["earnings", "rent", "in_area_1"]), + base.drop(columns=["earnings", "rent", "in_area_1"]), + ) + + @PROPERTY + @given(case=unit_frames()) + def test_encoding_is_deterministic(self, case) -> None: + tables, rule, _, _, group = case + first = _encode(tables, group) + second = _encode(tables, group) + pd.testing.assert_frame_equal( + first.tables[rule.entity], second.tables[rule.entity] + ) + + def test_unexecutable_group_bindings_are_deferred(self) -> None: + tables, _, _, _, group = _example() + encoded = _encode(tables, group) + assert {binding.engine_input for binding in encoded.deferred} == { + "child_age", + "tax_unit_income", + } + + def test_modules_filter_bindings(self) -> None: + bindings = [ + _binding( + "a", + ("fact:person.age",), + Identity(), + GroupRule.REFERENCE_MEMBER, + module="m/a.yaml", + canonical_input="zz:m/a#input.a", + ), + _binding( + "b", + ("fact:person.age",), + Identity(), + GroupRule.REFERENCE_MEMBER, + module="m/b.yaml", + canonical_input="zz:m/b#input.b", + ), + ] + read = {"fact:person.age"} + mapping = ConceptMapping( + engine="axiom:zz", + engine_version="0", + entity_correspondence={"person": "Person", "household": "Household"}, + input_declaration=InputDeclaration.USAGE_INFERRED, + bindings=tuple(bindings), + unmapped={item.id: "x" for item in CONCEPTS if item.id not in read}, + ) + tables, rule, _, _, group = _example() + encoded = mapping.encode_groups(tables, {"Family": group}, modules=["m/b.yaml"]) + assert list(encoded.tables[rule.entity].columns) == [rule.id_column, "b"] + + +def _example(): + rule = BenefitUnitRule( + entity="family", + dependent_child=DependentChildRule(max_age=17, financial_independence=None), + unparented_child="reference_person_unit", + split_parents="first_parent_unit", + provenance={"max_age": "test"}, + ) + rows = [ + (10, 1, 40, 11, None, 50_000.0, True), + (11, 1, 38, 10, None, 0.0, False), + (12, 1, 10, None, 10, 0.0, False), + (13, 1, 30, None, None, 26_000.0, False), + ] + person = pd.DataFrame( + rows, + columns=[ + "person_id", + "person_household_id", + "age", + "partner_person_id", + "parent_1_person_id", + "employment_income", + "receives_a", + ], + ) + person["parent_2_person_id"] = pd.array([None] * 4, dtype="Int64") + for column in ("partner_person_id", "parent_1_person_id"): + person[column] = pd.array(person[column].tolist(), dtype="Int64") + person["age"] = person["age"].astype(np.int64) + person["receives_b"] = False + person["interest_income"] = 0.0 + person["weeks_worked"] = np.int64(0) + person["has_disability"] = False + household = pd.DataFrame( + { + "household_id": [1], + "reference_person_id": pd.array([10], dtype="Int64"), + "tenure": ["rented_private"], + "rent": [52_000.0], + "mortgage_interest": [0.0], + "mortgage_principal": [0.0], + "area": np.asarray([1], dtype=np.int64), + } + ) + family, membership = build_benefit_units(person, household, rule) + group = benefit_unit_membership(person, family, membership, rule) + return {"person": person, "household": household}, rule, family, membership, group + + +class TestRefusals: + def test_a_membership_must_match_the_frame(self) -> None: + tables, rule, family, membership, group = _example() + cases = [ + ( + GroupMembership( + entity="family", + units=family, + person_unit=membership.iloc[:-1], + person_role=group.person_role.iloc[:-1], + ), + "align", + ), + ( + GroupMembership( + entity="family", + units=family, + person_unit=membership.replace({13: 99}), + person_role=group.person_role, + ), + "unknown family", + ), + ( + GroupMembership( + entity="family", + units=family, + person_unit=membership, + person_role=group.person_role.replace({"head": "partner"}), + ), + "exactly one head", + ), + ( + GroupMembership( + entity="family", + units=family.assign(family_household_id=7), + person_unit=membership, + person_role=group.person_role, + ), + "nest", + ), + ( + GroupMembership( + entity="family", + units=family.drop(columns="family_head_person_id"), + person_unit=membership, + person_role=group.person_role, + ), + "lacks", + ), + ] + for bad, message in cases: + with pytest.raises(ValueError, match=message): + MAPPING.encode_groups(tables, {"Family": bad}) + + def test_state_never_feeds_an_input_twice(self) -> None: + tables, _, _, _, group = _example() + twice = StateBinding( + engine_input="owner", + engine_entity="Family", + columns=("person.receives_a",), + test=StateTest.ANY_TRUE, + group_rule=GroupRule.ANY_MEMBER, + note="evidence", + ) + with pytest.raises(ValueError, match="concept and by state"): + MAPPING.encode_groups(tables, {"Family": group}, state_bindings=[twice]) + with pytest.raises(ValueError, match="twice by state"): + MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=[STATE[0], STATE[0]] + ) + + @pytest.mark.parametrize( + ("knobs", "message"), + [ + (GroupKnobs(zeroed=frozenset({"missing"})), "did not produce"), + (GroupKnobs(zeroed=frozenset({"owner"})), "not numeric"), + (GroupKnobs(allocation={"earnings": "per_adult_share"}), "not allocated"), + ( + GroupKnobs(state_columns={"household.nowhere": "household.area"}), + "no binding reads", + ), + ], + ) + def test_knobs_fail_closed(self, knobs, message) -> None: + tables, _, _, _, group = _example() + with pytest.raises(ValueError, match=message): + _encode(tables, group, knobs=knobs) + + @pytest.mark.parametrize( + ("change", "message"), + [ + ({"zeroed": {"a"}, "factors": {"a": 2.0}}, "both zeroed and scaled"), + ({"factors": {"a": -1.0}}, "non-negative"), + ({"factors": {"a": True}}, "number"), + ({"allocation": {"a": "evenly"}}, "evenly"), + ({"state_columns": {"household.a": "person.a"}}, "another entity"), + ({"state_columns": {"a": "household.a"}}, "state column"), + ], + ) + def test_malformed_knobs_are_refused(self, change, message) -> None: + with pytest.raises(ValueError, match=message): + GroupKnobs(**change) + + def test_state_columns_are_checked(self) -> None: + tables, _, _, _, group = _example() + person = tables["person"].assign(receives_a=1) + with pytest.raises(ValueError, match="boolean"): + _encode({**tables, "person": person}, group) + household = tables["household"].assign(area=["1"]) + with pytest.raises(ValueError, match="cannot equal"): + _encode({**tables, "household": household}, group) + with pytest.raises(ValueError, match="column 'area'"): + _encode( + {**tables, "household": tables["household"].drop(columns="area")}, group + ) + person = tables["person"].assign(receives_b=[True, None, False, False]) + with pytest.raises(ValueError, match="missing values"): + _encode({**tables, "person": person}, group) + + def test_one_input_name_is_one_computation(self) -> None: + tables, _, _, _, group = _example() + other_module = StateBinding( + engine_input="owner", + engine_entity="Family", + columns=("person.receives_a",), + test=StateTest.ANY_TRUE, + group_rule=GroupRule.ANY_MEMBER, + note="evidence", + module="other.yaml", + ) + with pytest.raises(ValueError, match="concept and by state"): + MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=[other_module] + ) + first = StateBinding.from_dict({**STATE[0].to_dict(), "module": "a.yaml"}) + same = StateBinding.from_dict({**STATE[0].to_dict(), "module": "b.yaml"}) + different = StateBinding.from_dict( + {**STATE[0].to_dict(), "module": "b.yaml", "negate": False} + ) + encoded = MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=[first, same] + ) + assert encoded.tables["family"]["non_recipient"].tolist() == [False, True] + with pytest.raises(ValueError, match="computed differently"): + MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=[first, different] + ) + + def test_an_allocation_knob_on_state_is_refused(self) -> None: + tables, _, _, _, group = _example() + with pytest.raises(ValueError, match="not allocated"): + _encode( + tables, + group, + knobs=GroupKnobs(allocation={"in_area_1": "per_adult_share"}), + ) + + def test_an_unseeded_take_up_program_is_left_out(self) -> None: + tables, rule, _, _, group = _example() + person = tables["person"].assign(take_up_seed=0.5) + tables = {**tables, "person": person} + binding = _binding( + "takes_up", + ("fact:person.take_up_seed",), + TakeUpThreshold(program="zz.x"), + GroupRule.ANY_MEMBER, + ) + mapping = _mapping([binding]) + encoded = mapping.encode_groups( + tables, {"Family": group}, take_up_rates={"zz.x": None} + ) + assert list(encoded.tables[rule.entity].columns) == [rule.id_column] + everyone = mapping.encode_groups( + tables, {"Family": group}, take_up_rates={"zz.x": 1.0} + ) + assert everyone.tables[rule.entity]["takes_up"].all() + + def test_memberships_align_by_index_and_ids(self) -> None: + tables, _, family, membership, group = _example() + shuffled = tables["person"].iloc[[0, 3, 2, 1]] + with pytest.raises(ValueError, match="align to the person table's index"): + MAPPING.encode_groups({**tables, "person": shuffled}, {"Family": group}) + realigned = GroupMembership( + entity="family", + units=family, + person_unit=membership.reindex(shuffled.index), + person_role=group.person_role.reindex(shuffled.index), + ) + first = MAPPING.encode_groups(tables, {"Family": group}).tables["family"] + second = MAPPING.encode_groups( + {**tables, "person": shuffled}, {"Family": realigned} + ).tables["family"] + pd.testing.assert_frame_equal(first, second) + fractional = GroupMembership( + entity="family", + units=family, + person_unit=membership.astype(np.float64) + 0.5, + person_role=group.person_role, + ) + with pytest.raises(ValueError, match="must be integers"): + MAPPING.encode_groups(tables, {"Family": fractional}) + + def test_roles_must_agree_with_the_partner_pointers(self) -> None: + tables, _, family, membership, group = _example() + # The couple's head is 10; claim the child 12 as the head's partner + # and demote the real partner 11 to a child. + roles = group.person_role.copy() + assert roles.tolist() == ["head", "partner", "dependent_child", "head"] + swapped = roles.copy() + swapped.iloc[1] = UnitRole.DEPENDENT_CHILD.value + swapped.iloc[2] = UnitRole.PARTNER.value + bad = GroupMembership( + entity="family", units=family, person_unit=membership, person_role=swapped + ) + with pytest.raises(ValueError, match="partner"): + MAPPING.encode_groups(tables, {"Family": bad}) + demoted = roles.copy() + demoted.iloc[1] = UnitRole.DEPENDENT_CHILD.value + bad = GroupMembership( + entity="family", units=family, person_unit=membership, person_role=demoted + ) + with pytest.raises(ValueError, match="must be its partner"): + MAPPING.encode_groups(tables, {"Family": bad}) + + def test_memberships_are_typed_and_distinct(self) -> None: + tables, _, _, _, group = _example() + with pytest.raises(ValueError, match="not a GroupMembership"): + MAPPING.encode_groups(tables, {"Family": object()}) + with pytest.raises(ValueError, match="Two engine entities"): + MAPPING.encode_groups(tables, {"Family": group, "TaxUnit": group}) + + def test_a_state_binding_needs_a_membership(self) -> None: + tables, _, _, _, group = _example() + other = StateBinding( + engine_input="x", + engine_entity="TaxUnit", + columns=("person.receives_a",), + test=StateTest.ANY_TRUE, + group_rule=GroupRule.ANY_MEMBER, + note="evidence", + ) + with pytest.raises(ValueError, match="'TaxUnit' membership"): + MAPPING.encode_groups(tables, {"Family": group}, state_bindings=[other]) + + def test_unit_composition_needs_membership_in_plain_encode(self) -> None: + # plain encode defers every group binding, unit composition included + tables, _, _, _, _ = _example() + deferred = {binding.engine_input for binding in MAPPING.encode(tables).deferred} + assert {f"unit_{feature.value}" for feature in UnitFeature} <= deferred + + +class TestConstruction: + @pytest.mark.parametrize( + ("binding", "message"), + [ + ( + dict(engine_entity="Person"), + "lives on an engine group entity", + ), + (dict(rule=GroupRule.ANY_MEMBER), "reference_member"), + (dict(concepts=("fact:person.age",)), "unit construction reads"), + ], + ) + def test_unit_composition_rules(self, binding, message) -> None: + fields = { + "engine_input": "flag", + "concepts": UNIT_CONCEPTS, + "transform": UnitComposition(feature=UnitFeature.SOLE_PARENT), + "rule": GroupRule.REFERENCE_MEMBER, + **binding, + } + entity = fields.pop("engine_entity", "Family") + rule = fields.pop("rule") + if entity == "Person": + rule = None + with pytest.raises(ValueError, match=message): + _mapping([_binding(engine_entity=entity, rule=rule, **fields)]) + + @pytest.mark.parametrize( + ("concepts", "message"), + [ + (("fact:person.employment_income",), "two or more"), + (("fact:person.age", "fact:person.weeks_worked"), "not an amount"), + ( + ("fact:person.employment_income", "fact:household.rent"), + "one entity", + ), + ], + ) + def test_scaled_sums_add_amounts_on_one_entity(self, concepts, message) -> None: + with pytest.raises(ValueError, match=message): + _binding("x", concepts, ScaledSum(factor=WEEK), GroupRule.SUM_OVER_MEMBERS) + + @pytest.mark.parametrize("factor", [0.0, -1.0, float("inf"), float("nan")]) + def test_scaled_sum_factors_are_positive(self, factor) -> None: + with pytest.raises(ValueError, match="positive"): + ScaledSum(factor=factor) + + def test_a_scaled_sum_on_a_plain_input_encodes(self) -> None: + binding = InputBinding( + engine_input="weekly", + engine_entity="Person", + concepts=("fact:person.employment_income", "fact:person.interest_income"), + transform=ScaledSum(factor=WEEK), + relation=AlignmentRelation.APPROXIMATE, + note="evidence", + ) + tables, _, _, _, _ = _example() + encoded = _mapping([binding]).encode(tables) + person = tables["person"] + np.testing.assert_allclose( + encoded.tables["person"]["weekly"], + (person["employment_income"] + person["interest_income"]) * WEEK, + ) + assert binding.ref not in {b.ref for b in encoded.deferred} + assert ( + "fact:person.employment_income" + not in _mapping([binding]).invertible_concepts() + ) + + +class TestSerialization: + @pytest.mark.parametrize( + "transform", + [ScaledSum(factor=WEEK), *(UnitComposition(feature=f) for f in UnitFeature)], + ) + def test_new_transforms_round_trip(self, transform) -> None: + assert transform_from_dict( + json.loads(json.dumps(transform_to_dict(transform))) + ) == (transform) + + @pytest.mark.parametrize( + "data", + [ + {"kind": "scaled_sum"}, + {"kind": "scaled_sum", "factor": "1"}, + {"kind": "scaled_sum", "factor": True}, + {"kind": "scaled_sum", "factor": 1, "x": 1}, + {"kind": "unit_composition"}, + {"kind": "unit_composition", "feature": "orphan"}, + {"kind": "unit_composition", "feature": 3}, + ], + ) + def test_malformed_new_transforms_raise_value_error(self, data) -> None: + with pytest.raises(ValueError): + transform_from_dict(data) + + @pytest.mark.parametrize("binding", STATE) + def test_state_bindings_round_trip(self, binding) -> None: + payload = json.loads(json.dumps(binding.to_dict())) + assert StateBinding.from_dict(payload) == binding + + @pytest.mark.parametrize( + ("change", "message"), + [ + ({"columns": []}, "distinct columns"), + ({"columns": ["person.a", "household.b"]}, "one entity"), + ({"columns": ["family.a"]}, "state column"), + ({"group_rule": "household_value"}, "collapses by"), + ({"test": "equals"}, "compares one column"), + ({"value": 1}, "takes no value"), + ({"note": ""}, "note"), + ({"negate": 1}, "negate"), + ({"test": "equals", "columns": ["person.a"], "value": 1.5}, "exact"), + ({"group_rule": "sum_over_members"}, "collapses by"), + ({"extra": 1}, "unexpected"), + ], + ) + def test_malformed_state_bindings_raise_value_error(self, change, message) -> None: + payload = {**STATE[0].to_dict(), **change} + with pytest.raises(ValueError, match=message): + StateBinding.from_dict(payload) + + @settings(max_examples=200, deadline=None) + @given( + data=st.recursive( + st.none() | st.booleans() | st.integers() | st.text(max_size=5), + lambda inner: ( + st.lists(inner, max_size=3) + | st.dictionaries( + st.sampled_from( + [ + "engine_input", + "engine_entity", + "columns", + "test", + "group_rule", + "note", + "value", + ] + ) + | st.text(max_size=4), + inner, + max_size=6, + ) + ), + max_leaves=10, + ) + ) + def test_arbitrary_state_payloads_only_raise_value_error(self, data) -> None: + with pytest.raises(ValueError): + StateBinding.from_dict(data) + + +class TestNewZealandMapping: + AS = "nz/statutes/social_security/accommodation_supplement/core.yaml" + NZ_RULE = BenefitUnitRule( + entity="family", + dependent_child=DependentChildRule(max_age=17, financial_independence=None), + unparented_child="reference_person_unit", + split_parents="first_parent_unit", + provenance={"max_age": "test"}, + ) + + @PROPERTY + @given(tables=concept_frames(max_households=4, max_members=6)) + def test_every_accommodation_supplement_group_binding_executes( + self, tables + ) -> None: + mapping = axiom_concept_mapping("nz") + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, self.NZ_RULE) + group = benefit_unit_membership(person, family, membership, self.NZ_RULE) + encoded = mapping.encode_groups(tables, {"Family": group}, modules=[self.AS]) + expected = { + binding.engine_input + for binding in mapping.bindings + if binding.module == self.AS + } + table = encoded.tables["family"] + assert set(table.columns) == {"family_id", *expected} + assert encoded.deferred == () + for binding in mapping.bindings: + if binding.module != self.AS: + continue + kind = table[binding.engine_input].dtype.kind + assert kind in ("b", "f"), binding.engine_input + rent = table["accommodation_supplement_weekly_rent_paid"].sum() + np.testing.assert_allclose(rent, household["rent"].sum() * WEEK, rtol=1e-12) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py b/packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py new file mode 100644 index 0000000000..ab849fea22 --- /dev/null +++ b/packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py @@ -0,0 +1,599 @@ +"""The input closure: every root input of a bound module closed exactly once. + +The closure's checks run against a toy mapping and a toy engine surface, so +each way a closure can disagree with them is shown to be reported; the +country closures are checked against their real surfaces where they live. +""" + +import json +from types import MappingProxyType + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from microcosm.frame.adapters.axiom_input_surface import ( + AxiomInputSurface, + AxiomModuleSurface, +) +from microcosm.frame.concept_mapping import ( + Allocation, + ConceptMapping, + GroupRule, + Identity, + InputBinding, + InputDeclaration, + InputRef, + StateBinding, + StateTest, +) +from microcosm.frame.concepts import CONCEPTS, AlignmentRelation +from microcosm.frame.input_closure import ( + INPUT_CLOSURE_FORMAT, + ClosureClass, + ClosureEntry, + EncodedBy, + InputClosure, + Knob, + KnobKind, + UndeterminedAction, + UndeterminedPolicy, + apply_defaults, + resolve_judgments, +) + +MODULE = "zz/m.yaml" +SHA = "0" * 64 +SURFACE = AxiomInputSurface( + country="zz", + rulespec_commit="r1", + engine_commit="e1", + engine_surface="dense_root_inputs", + modules=MappingProxyType( + { + MODULE: AxiomModuleSurface( + path=MODULE, + sha256=SHA, + status="compiled", + inputs=MappingProxyType( + {"Family": ("age", "area_1", "bridge", "flag")} + ), + canonical_inputs=MappingProxyType({}), + ) + } + ), +) +BOUND = InputBinding( + engine_input="age", + engine_entity="Family", + concepts=("fact:person.age",), + transform=Identity(), + relation=AlignmentRelation.APPROXIMATE, + note="evidence", + group_rule=GroupRule.REFERENCE_MEMBER, + module=MODULE, + canonical_input="zz:m#input.age", +) +MAPPING = ConceptMapping( + engine="axiom:zz", + engine_version="r1", + entity_correspondence={"person": "Person", "household": "Household"}, + input_declaration=InputDeclaration.USAGE_INFERRED, + bindings=(BOUND,), + unmapped={item.id: "x" for item in CONCEPTS if item.id != "fact:person.age"}, +) +AREA = StateBinding( + engine_input="area_1", + engine_entity="Family", + columns=("household.area",), + test=StateTest.EQUALS, + value=1, + group_rule=GroupRule.HOUSEHOLD_VALUE, + note="evidence", + module=MODULE, +) +ENTRIES = ( + ClosureEntry( + module=MODULE, + entity="Family", + input="age", + closure_class=ClosureClass.ENCODED, + encoded_by=EncodedBy.CONCEPT_BINDING, + ), + ClosureEntry( + module=MODULE, + entity="Family", + input="area_1", + closure_class=ClosureClass.ENCODED, + encoded_by=EncodedBy.STATE_BINDING, + state_binding=AREA, + reason="model state", + ), + ClosureEntry( + module=MODULE, + entity="Family", + input="bridge", + closure_class=ClosureClass.ENCODED, + encoded_by=EncodedBy.GRAPH_NODE, + node="zz.bridge", + reason="engine-computed elsewhere", + ), + ClosureEntry( + module=MODULE, + entity="Family", + input="flag", + closure_class=ClosureClass.DEFAULTED, + value=False, + reason="not observed", + citation="Act s 1", + new_default=True, + ), +) +KNOBS = ( + Knob( + name="area_column", + kind=KnobKind.STATE_COLUMN, + targets=("household.area",), + central="area", + values=("area", "area_alt"), + method_card_row="MC9", + note="which area", + ), + Knob( + name="age_factor", + kind=KnobKind.FACTOR, + targets=("age",), + central=1.0, + values=(), + method_card_row="MC0", + note="scale", + ), +) + + +def _closure(**change) -> InputClosure: + fields = { + "country": "zz", + "content_basis": "transport", + "mapping_engine": "axiom:zz", + "mapping_engine_version": "r1", + "surface_rulespec_commit": "r1", + "surface_engine_commit": "e1", + "modules": {MODULE: SHA}, + "engine_optional_evidence": "the engine has no optional inputs", + "undetermined": UndeterminedPolicy( + action=UndeterminedAction.REFUSE, + reason="none can arise", + alternatives=(UndeterminedAction.NOT_ELIGIBLE,), + new_default=True, + ), + "knobs": KNOBS, + "entries": ENTRIES, + **change, + } + return InputClosure(**fields) + + +class TestCheck: + def test_a_complete_closure_holds(self) -> None: + closure = _closure() + assert closure.check(MAPPING, SURFACE) == () + assert closure.defaults(MODULE, "Family") == {"flag": False} + assert closure.state_bindings() == (AREA,) + assert closure.state_bindings(["other.yaml"]) == () + assert closure.graph_node_inputs() == { + InputRef("bridge", "Family", MODULE): "zz.bridge" + } + assert [entry.input for entry in closure.new_defaults()] == ["flag"] + + def test_an_unclosed_input_is_reported(self) -> None: + closure = _closure(entries=ENTRIES[:-1]) + assert closure.check(MAPPING, SURFACE) == ( + f"flag (Family, {MODULE}) is not closed.", + ) + + def test_a_closed_name_the_surface_lacks_is_reported(self) -> None: + stray = ClosureEntry( + module=MODULE, + entity="Person", + input="flag", + closure_class=ClosureClass.DEFAULTED, + value=0, + reason="r", + citation="c", + ) + (problem,) = _closure(entries=(*ENTRIES, stray)).check(MAPPING, SURFACE) + assert "closed but not a root input" in problem + + def test_concept_encoding_must_be_the_mappings_binding(self) -> None: + defaulted_age = ClosureEntry( + module=MODULE, + entity="Family", + input="age", + closure_class=ClosureClass.DEFAULTED, + value=30, + reason="r", + citation="c", + ) + (problem,) = _closure( + entries=(defaulted_age, *ENTRIES[1:]), knobs=KNOBS[:1] + ).check(MAPPING, SURFACE) + assert "bound by the mapping but closed as defaulted" in problem + unbound = ConceptMapping( + **{ + **{ + name: getattr(MAPPING, name) + for name in ( + "engine", + "engine_version", + "entity_correspondence", + "input_declaration", + ) + }, + "bindings": (), + "unmapped": {item.id: "x" for item in CONCEPTS}, + } + ) + (problem,) = _closure().check(unbound, SURFACE) + assert "concept-encoded but not bound" in problem + awaiting = ( + ClosureEntry( + module=MODULE, + entity="Family", + input="age", + closure_class=ClosureClass.ENCODED, + encoded_by=EncodedBy.CONCEPT_BINDING, + awaiting="a later change", + ), + *ENTRIES[1:], + ) + assert _closure(entries=awaiting).check(unbound, SURFACE) == () + (problem,) = _closure(entries=awaiting).check(MAPPING, SURFACE) + assert "no longer awaits a later change" in problem + + def test_a_bound_input_must_be_closed_even_off_the_surface(self) -> None: + ghost = InputBinding( + engine_input="ghost", + engine_entity="Family", + concepts=("fact:person.age",), + transform=Identity(), + relation=AlignmentRelation.APPROXIMATE, + note="evidence", + group_rule=GroupRule.REFERENCE_MEMBER, + module=MODULE, + canonical_input="zz:m#input.ghost", + ) + mapping = ConceptMapping( + **{ + name: getattr(MAPPING, name) + for name in ( + "engine", + "engine_version", + "entity_correspondence", + "input_declaration", + "unmapped", + ) + }, + bindings=(BOUND, ghost), + ) + (problem,) = _closure().check(mapping, SURFACE) + assert ( + problem + == f"ghost (Family, {MODULE}) is bound by the mapping but not closed." + ) + + def test_a_binding_transport_drops_cannot_encode_a_transport_run(self) -> None: + receipt = InputBinding( + engine_input="age", + engine_entity="Family", + concepts=("fact:person.public_pension_income",), + transform=Identity(), + relation=AlignmentRelation.APPROXIMATE, + note="evidence", + group_rule=GroupRule.SUM_OVER_MEMBERS, + module=MODULE, + canonical_input="zz:m#input.age", + ) + mapping = ConceptMapping( + **{ + name: getattr(MAPPING, name) + for name in ( + "engine", + "engine_version", + "entity_correspondence", + "input_declaration", + ) + }, + bindings=(receipt,), + unmapped={ + item.id: "x" + for item in CONCEPTS + if item.id != "fact:person.public_pension_income" + }, + ) + (problem,) = _closure().check(mapping, SURFACE) + assert "reads a concept transport frames drop" in problem + defaulted_age = ClosureEntry( + module=MODULE, + entity="Family", + input="age", + closure_class=ClosureClass.DEFAULTED, + value=0, + reason="the donor's receipt is dropped", + citation="Act s 2", + ) + closure = _closure(entries=(defaulted_age, *ENTRIES[1:]), knobs=KNOBS[:1]) + assert closure.check(mapping, SURFACE) == () + own = _closure(content_basis="own_data") + assert own.check(mapping, SURFACE) == () + + def test_pins_are_checked(self) -> None: + problems = _closure( + mapping_engine_version="r0", + surface_rulespec_commit="r0", + surface_engine_commit="e0", + modules={MODULE: "1" * 64}, + ).check(MAPPING, SURFACE) + assert len(problems) == 4 + (problem,) = _closure(modules={MODULE: SHA, "zz/absent.yaml": SHA}).check( + MAPPING, SURFACE + ) + assert "not in the input surface" in problem + + +class TestConstruction: + @pytest.mark.parametrize( + ("change", "message"), + [ + ({"closure_class": "encoded"}, "says what encodes it"), + ( + { + "closure_class": "encoded", + "encoded_by": "graph_node", + "value": None, + "citation": None, + }, + "names its node", + ), + ({"value": None}, "boolean or a number"), + ({"value": "x"}, "boolean or a number"), + ({"value": float("nan")}, "finite"), + ({"citation": None}, "cites"), + ({"reason": ""}, "needs a reason"), + ({"input": ""}, "names its module"), + ], + ) + def test_malformed_entries_are_refused(self, change, message) -> None: + fields = { + "module": MODULE, + "entity": "Family", + "input": "flag", + "closure_class": ClosureClass.DEFAULTED, + "value": False, + "reason": "r", + "citation": "c", + **change, + } + with pytest.raises(ValueError, match=message): + ClosureEntry(**fields) + + def test_only_defaults_carry_values(self) -> None: + with pytest.raises(ValueError, match="only a default"): + ClosureEntry( + module=MODULE, + entity="Family", + input="age", + closure_class=ClosureClass.ENCODED, + encoded_by=EncodedBy.CONCEPT_BINDING, + value=1, + ) + + def test_a_state_entry_carries_its_own_binding(self) -> None: + with pytest.raises(ValueError, match="feeds another input"): + ClosureEntry( + module=MODULE, + entity="Family", + input="flag", + closure_class=ClosureClass.ENCODED, + encoded_by=EncodedBy.STATE_BINDING, + state_binding=AREA, + reason="r", + ) + with pytest.raises(ValueError, match="carries its binding"): + ClosureEntry( + module=MODULE, + entity="Family", + input="area_1", + closure_class=ClosureClass.ENCODED, + encoded_by=EncodedBy.STATE_BINDING, + reason="r", + ) + + def test_closures_refuse_duplicates_and_stray_knobs(self) -> None: + with pytest.raises(ValueError, match="closed twice"): + _closure(entries=(*ENTRIES, ENTRIES[-1])) + with pytest.raises(ValueError, match="does not list"): + _closure(modules={}) + with pytest.raises(ValueError, match="no encoded entry"): + _closure( + knobs=( + Knob( + name="off", + kind=KnobKind.SWITCH_ZEROES, + targets=("flag",), + central=True, + values=(), + method_card_row="MC0", + note="n", + ), + ) + ) + with pytest.raises(ValueError, match="unique"): + _closure(knobs=(KNOBS[0], KNOBS[0])) + + @pytest.mark.parametrize( + ("change", "message"), + [ + ({"central": "elsewhere"}, "takes"), + ({"values": ()}, "lists the values"), + ({"targets": ("household.a", "household.b")}, "one state column"), + ({"kind": "factor", "values": ("x",)}, "no value list"), + ({"kind": "factor", "values": (), "central": -1.0}, "non-negative"), + ({"kind": "switch_zeroes", "values": (), "central": 1}, "true or false"), + ( + {"kind": "allocation", "values": ("evenly",), "central": "evenly"}, + "evenly", + ), + ({"note": ""}, "needs a name"), + ], + ) + def test_malformed_knobs_are_refused(self, change, message) -> None: + fields = {**KNOBS[0].to_dict(), **change} + fields["kind"] = KnobKind(fields["kind"]) + with pytest.raises(ValueError, match=message): + Knob(**fields) + + +class TestKnobs: + def test_central_values_change_nothing(self) -> None: + knobs = _closure().group_knobs() + assert knobs.targets() == frozenset() + assert dict(knobs.state_columns) == {} + assert dict(knobs.factors) == {} + assert knobs.zeroed == frozenset() + + def test_scenario_values_translate(self) -> None: + allocation = Knob( + name="rent_allocation", + kind=KnobKind.ALLOCATION, + targets=("age",), + central="reference_unit", + values=("reference_unit", "per_adult_share"), + method_card_row="MC8", + note="n", + ) + switch = Knob( + name="test_on", + kind=KnobKind.SWITCH_ZEROES, + targets=("bridge",), + central=True, + values=(), + method_card_row="MC10", + note="n", + ) + closure = _closure(knobs=(*KNOBS, allocation, switch)) + knobs = closure.group_knobs( + { + "area_column": "area_alt", + "age_factor": 2, + "rent_allocation": "per_adult_share", + "test_on": False, + } + ) + assert dict(knobs.state_columns) == {"household.area": "household.area_alt"} + assert dict(knobs.factors) == {"age": 2.0} + assert dict(knobs.allocation) == {"age": Allocation.PER_ADULT_SHARE} + assert knobs.zeroed == frozenset({"bridge"}) + with pytest.raises(ValueError, match="Unknown knobs"): + closure.group_knobs({"nope": 1}) + with pytest.raises(ValueError, match="takes"): + closure.group_knobs({"area_column": "area_x"}) + + +class TestSerialization: + def test_closures_round_trip_through_json(self) -> None: + closure = _closure() + payload = json.loads(json.dumps(closure.to_dict())) + assert payload["format"] == INPUT_CLOSURE_FORMAT + again = InputClosure.from_dict(payload) + assert again.to_dict() == closure.to_dict() + assert again.check(MAPPING, SURFACE) == () + + def test_review_fields_beside_the_closure_are_allowed(self) -> None: + payload = {**_closure().to_dict(), "description": "notes", "status": "draft"} + payload["input_surface"]["fixture"] = "path/to/surface.json" + assert InputClosure.from_dict(payload).to_dict() == _closure().to_dict() + + @pytest.mark.parametrize( + "change", + [ + {"format": "x"}, + {"rulespec_paths": {"m": 1}}, + {"knobs": {}}, + {"entries": [{"module": "m"}]}, + {"undetermined_judgments": {"action": "guess"}}, + {"mapping": {"engine": "x"}}, + {"engine_optional_evidence": 3}, + {"content_basis": "survey"}, + {"entries": [{**ENTRIES[1].to_dict(), "state_binding": {"module": "x"}}]}, + ], + ) + def test_malformed_closures_raise_value_error(self, change) -> None: + payload = {**_closure().to_dict(), **change} + with pytest.raises(ValueError): + InputClosure.from_dict(payload) + + @settings(max_examples=200, deadline=None) + @given( + data=st.recursive( + st.none() | st.booleans() | st.integers() | st.text(max_size=5), + lambda inner: ( + st.lists(inner, max_size=3) + | st.dictionaries(st.text(max_size=6), inner, max_size=4) + ), + max_leaves=10, + ) + ) + def test_arbitrary_payloads_only_raise_value_error(self, data) -> None: + for reader in (InputClosure.from_dict, ClosureEntry.from_dict, Knob.from_dict): + with pytest.raises(ValueError): + reader(data) + + +class TestJudgments: + @settings(max_examples=200, deadline=None) + @given( + codes=st.lists(st.sampled_from([-1, 0, 1]), min_size=1, max_size=40), + action=st.sampled_from( + [UndeterminedAction.NOT_ELIGIBLE, UndeterminedAction.ELIGIBLE] + ), + ) + def test_determined_codes_map_exactly(self, codes, action) -> None: + values = np.asarray(codes, dtype=np.int64) + eligible = resolve_judgments(values, action) + assert eligible.dtype == bool + assert eligible[values == 1].all() + assert not eligible[values == -1].any() + assert (eligible[values == 0] == (action is UndeterminedAction.ELIGIBLE)).all() + lower = resolve_judgments(values, UndeterminedAction.NOT_ELIGIBLE) + upper = resolve_judgments(values, UndeterminedAction.ELIGIBLE) + assert (lower <= upper).all() + + def test_no_codes_resolve_to_no_entitlement(self) -> None: + for action in UndeterminedAction: + assert resolve_judgments(np.asarray([]), action).tolist() == [] + + def test_refuse_fails_on_an_undetermined_code_only(self) -> None: + codes = np.asarray([1, -1, 1], dtype=np.int8) + assert resolve_judgments(codes, "refuse").tolist() == [True, False, True] + with pytest.raises(ValueError, match="1 judgment"): + resolve_judgments(np.asarray([1, 0]), UndeterminedAction.REFUSE) + + @pytest.mark.parametrize("codes", [[2], [0.5], [True]]) + def test_other_codes_are_refused(self, codes) -> None: + with pytest.raises(ValueError, match="-1, 0 or 1"): + resolve_judgments(np.asarray(codes), UndeterminedAction.NOT_ELIGIBLE) + + +class TestDefaults: + def test_defaults_become_constant_columns(self) -> None: + table = pd.DataFrame({"family_id": [1, 2, 3]}) + out = apply_defaults(table, {"flag": False, "count": 0, "amount": 2.5}) + assert out["flag"].dtype == bool and not out["flag"].any() + assert out["count"].dtype.kind == "i" and (out["count"] == 0).all() + assert out["amount"].dtype == np.float64 + assert list(table.columns) == ["family_id"] + with pytest.raises(ValueError, match="already encoded"): + apply_defaults(out, {"flag": True}) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py new file mode 100644 index 0000000000..8fbc60449c --- /dev/null +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -0,0 +1,488 @@ +"""Benefit units built from concept pointers: the invariants, as properties. + +Every property runs over generated valid concept frames +(:func:`test_support.microcosm_frame.concept_frames.concept_frames`) and +generated rules, so a unit rule is checked against every pointer shape the +schema allows rather than a few hand-picked households: + +- partition: every person is in exactly one unit, and every unit has members; +- every unit has an adult (its head) and at most one partner; +- partners share a unit; +- every dependent child shares a unit with a co-resident parent, or with the + household's reference person when it has none; +- units nest in households; +- family weights are inherited: the sum of family weights equals the sum of + household weight times units per household; +- unit ids are deterministic and independent of row order. +""" + +import json + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.frame.concept_mapping import UnitRole +from microcosm.frame.unit_construction import ( + BENEFIT_UNIT_RULE_FORMAT, + BenefitUnitRule, + DependentChildRule, + FinancialIndependenceTest, + SplitParentsPolicy, + UnparentedChildPlacement, + benefit_unit_attributes, + benefit_unit_membership, + benefit_unit_roles, + build_benefit_units, + units_per_household, +) +from test_support.microcosm_frame.concept_frames import concept_frames +from test_support.microcosm_frame.unit_rules import unit_rules + +PROPERTY = settings(max_examples=150, deadline=None) + + +def _rows(person: pd.DataFrame, column: str) -> np.ndarray: + ids = pd.Index(person["person_id"].to_numpy()) + values = person[column] + out = np.full(len(person), -1, dtype=np.int64) + present = values.notna().to_numpy() + out[present] = ids.get_indexer(values[present].to_numpy(dtype=np.int64)) + return out + + +def _reference_rows(person: pd.DataFrame, household: pd.DataFrame) -> np.ndarray: + ids = pd.Index(person["person_id"].to_numpy()) + by_household = pd.Series( + ids.get_indexer(household["reference_person_id"].to_numpy(dtype=np.int64)), + index=household["household_id"].to_numpy(), + ) + return by_household.reindex(person["person_household_id"].to_numpy()).to_numpy() + + +class TestInvariants: + @PROPERTY + @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + def test_units_partition_persons_and_nest_in_households(self, tables, rule) -> None: + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, rule) + ids = family[rule.id_column] + # partition: one unit per person, every unit has members + assert membership.index.equals(person.index) + assert membership.notna().all() + assert membership.isin(ids).all() + assert set(membership) == set(ids) + assert ids.is_unique + # nesting: members live in the household the unit declares + declared = pd.Series( + family[rule.household_column].to_numpy(), index=ids.to_numpy() + ) + assert np.array_equal( + declared.reindex(membership.to_numpy()).to_numpy(), + person["person_household_id"].to_numpy(), + ) + # Frame group tables are sorted by id, with int64 structure columns + assert ids.is_monotonic_increasing + for column in (rule.id_column, rule.household_column, rule.head_column): + assert family[column].dtype == np.int64 + assert membership.dtype == np.int64 + + @PROPERTY + @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + def test_every_unit_has_an_adult_and_partners_share_it(self, tables, rule) -> None: + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, rule) + roles = benefit_unit_roles(person, family, membership, rule) + heads = pd.Series( + family[rule.head_column].to_numpy(), index=family[rule.id_column] + ) + unit_of = pd.Series(membership.to_numpy(), index=person["person_id"].to_numpy()) + # the head is a member, the only head, and an adult + assert np.array_equal( + unit_of.reindex(heads.to_numpy()).to_numpy(), heads.index.to_numpy() + ) + role_counts = pd.crosstab(membership.to_numpy(), roles.to_numpy()).reindex( + columns=[role.value for role in UnitRole], fill_value=0 + ) + assert (role_counts[UnitRole.HEAD.value] == 1).all() + assert (role_counts[UnitRole.PARTNER.value] <= 1).all() + partner = _rows(person, "partner_person_id") + paired = partner >= 0 + assert np.array_equal( + membership.to_numpy()[paired], membership.to_numpy()[partner[paired]] + ) + assert not roles[paired].eq(UnitRole.DEPENDENT_CHILD.value).any() + + @PROPERTY + @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + def test_dependent_children_live_with_a_parent_or_caregiver( + self, tables, rule + ) -> None: + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, rule) + roles = benefit_unit_roles(person, family, membership, rule).to_numpy() + unit = membership.to_numpy() + parent = _rows(person, "parent_1_person_id") + reference = _reference_rows(person, household) + child = roles == UnitRole.DEPENDENT_CHILD.value + with_parent = child & (parent >= 0) + assert np.array_equal(unit[with_parent], unit[parent[with_parent]]) + without = child & (parent < 0) + assert np.array_equal(unit[without], unit[reference[without]]) + # a dependent child meets the rule; partners and parents never do + ages = person["age"].to_numpy() + assert (ages[child] <= rule.dependent_child.max_age).all() + assert person["partner_person_id"][child].isna().all() + named = set(person["parent_1_person_id"].dropna()) | set( + person["parent_2_person_id"].dropna() + ) + assert not person["person_id"][child].isin(named).any() + test = rule.dependent_child.financial_independence + if test is not None: + hours = person["usual_weekly_hours"].to_numpy() + independent = (ages >= test.min_age) & ( + hours >= test.min_usual_weekly_hours + ) + assert not (independent & child).any() + if rule.unparented_child is UnparentedChildPlacement.OWN_UNIT: + assert not without.any() + + @PROPERTY + @given( + tables=concept_frames(max_households=5, max_members=6), + rule=unit_rules(), + weights=st.lists( + st.floats(min_value=0.001, max_value=1e6, allow_nan=False), + min_size=5, + max_size=5, + ), + ) + def test_family_weights_are_inherited_from_households( + self, tables, rule, weights + ) -> None: + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, rule) + household_weights = np.asarray(weights[: len(household)], dtype=np.float64) + order = np.argsort(household["household_id"].to_numpy()) + frame_person = person.loc[:, ["person_id", "person_household_id"]].assign( + **{rule.membership_column: membership.to_numpy()} + ) + frame = Frame( + schema=EntitySchema(group_entities=("household", rule.entity)), + tables={ + "person": frame_person, + "household": household.loc[:, ["household_id"]] + .iloc[order] + .reset_index(drop=True), + rule.entity: family.loc[:, [rule.id_column]], + }, + weights={ + "household": Weights( + values=household_weights[order], kind=WeightKind.DESIGN + ) + }, + ) + family_weights = frame.resolve_weights(rule.entity).values + counts = units_per_household(household, family, rule).to_numpy() + np.testing.assert_allclose( + family_weights.sum(), (household_weights * counts).sum(), rtol=1e-12 + ) + by_household = pd.Series(household_weights, index=household["household_id"]) + np.testing.assert_array_equal( + family_weights, + by_household.reindex(family[rule.household_column]).to_numpy(), + ) + + @PROPERTY + @given( + tables=concept_frames(max_households=5, max_members=6), + rule=unit_rules(), + data=st.data(), + ) + def test_units_do_not_depend_on_row_order(self, tables, rule, data) -> None: + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, rule) + order = data.draw(st.permutations(range(len(person)))) + shuffled = person.iloc[list(order)].reset_index(drop=True) + family_2, membership_2 = build_benefit_units(shuffled, household, rule) + pd.testing.assert_frame_equal(family, family_2) + first = pd.Series(membership.to_numpy(), index=person["person_id"].to_numpy()) + second = pd.Series( + membership_2.to_numpy(), index=shuffled["person_id"].to_numpy() + ) + pd.testing.assert_series_equal(first.sort_index(), second.sort_index()) + + @PROPERTY + @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + def test_attributes_count_the_roles(self, tables, rule) -> None: + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, rule) + attributes = benefit_unit_attributes(person, family, membership, rule) + roles = benefit_unit_roles(person, family, membership, rule) + counts = ( + pd.crosstab(membership.to_numpy(), roles.to_numpy()) + .reindex(columns=[role.value for role in UnitRole], fill_value=0) + .reindex(family[rule.id_column], fill_value=0) + ) + children = counts[UnitRole.DEPENDENT_CHILD.value].to_numpy() + assert np.array_equal(attributes["n_dependent_children"], children) + assert (attributes["n_adults"].between(1, 2)).all() + assert np.array_equal( + attributes["n_members"], attributes["n_adults"] + children + ) + assert set(attributes["family_type"]) <= { + "single", + "couple", + "sole_parent", + "couple_with_children", + } + assert ( + attributes["youngest_dependent_child_age"] + .isna() + .eq(attributes["n_dependent_children"] == 0) + .all() + ) + membership_view = benefit_unit_membership(person, family, membership, rule) + assert membership_view.entity == rule.entity + pd.testing.assert_series_equal(membership_view.person_role, roles) + + +class TestPlacement: + RULE = BenefitUnitRule( + entity="family", + dependent_child=DependentChildRule(max_age=17, financial_independence=None), + unparented_child="reference_person_unit", + split_parents="refuse", + provenance={"max_age": "test"}, + ) + + @staticmethod + def _frame(rows, reference) -> tuple[pd.DataFrame, pd.DataFrame]: + person = pd.DataFrame( + rows, + columns=[ + "person_id", + "person_household_id", + "age", + "partner_person_id", + "parent_1_person_id", + "parent_2_person_id", + "usual_weekly_hours", + ], + ) + for column in ("partner_person_id", "parent_1_person_id", "parent_2_person_id"): + person[column] = pd.array(person[column].tolist(), dtype="Int64") + person["age"] = person["age"].astype(np.int64) + household = pd.DataFrame( + { + "household_id": sorted(reference), + "reference_person_id": pd.array( + [reference[key] for key in sorted(reference)], dtype="Int64" + ), + } + ) + return person, household + + def test_a_couple_their_children_and_an_adult_child(self) -> None: + person, household = self._frame( + [ + (10, 1, 40, 11, None, None, 40.0), + (11, 1, 38, 10, None, None, 0.0), + (12, 1, 10, None, 10, 11, 0.0), + (13, 1, 22, None, 10, None, 40.0), + ], + {1: 11}, + ) + family, membership = build_benefit_units(person, household, self.RULE) + assert family["family_id"].tolist() == [11, 13] + assert family["family_head_person_id"].tolist() == [11, 13] + assert membership.tolist() == [11, 11, 11, 13] + roles = benefit_unit_roles(person, family, membership, self.RULE) + assert roles.tolist() == ["partner", "head", "dependent_child", "head"] + + def test_a_parentless_child_joins_the_reference_unit(self) -> None: + person, household = self._frame( + [(1, 1, 60, None, None, None, 0.0), (2, 1, 12, None, None, None, 0.0)], + {1: 1}, + ) + _, membership = build_benefit_units(person, household, self.RULE) + assert membership.tolist() == [1, 1] + own = BenefitUnitRule( + **{**self._fields(), "unparented_child": UnparentedChildPlacement.OWN_UNIT} + ) + _, membership = build_benefit_units(person, household, own) + assert membership.tolist() == [1, 2] + + def test_a_lone_young_reference_person_heads_their_unit(self) -> None: + person, household = self._frame( + [(1, 1, 17, None, None, None, 0.0), (2, 1, 15, None, None, None, 0.0)], + {1: 1}, + ) + family, membership = build_benefit_units(person, household, self.RULE) + assert membership.tolist() == [1, 1] + attributes = benefit_unit_attributes(person, family, membership, self.RULE) + assert attributes["family_type"].tolist() == ["sole_parent"] + refuse = BenefitUnitRule( + **{**self._fields(), "unparented_child": UnparentedChildPlacement.REFUSE} + ) + with pytest.raises(ValueError, match="1 candidate"): + build_benefit_units(person, household, refuse) + + def test_a_teen_parent_heads_their_own_unit(self) -> None: + person, household = self._frame( + [ + (1, 1, 45, None, None, None, 0.0), + (2, 1, 16, None, 1, None, 0.0), + (3, 1, 0, None, 2, None, 0.0), + ], + {1: 1}, + ) + _, membership = build_benefit_units(person, household, self.RULE) + assert membership.tolist() == [1, 2, 2] + + def test_financial_independence_takes_a_worker_out_of_dependency(self) -> None: + person, household = self._frame( + [(1, 1, 45, None, None, None, 0.0), (2, 1, 17, None, 1, None, 40.0)], + {1: 1}, + ) + _, membership = build_benefit_units(person, household, self.RULE) + assert membership.tolist() == [1, 1] + working = BenefitUnitRule( + **{ + **self._fields(), + "dependent_child": DependentChildRule( + max_age=17, + financial_independence=FinancialIndependenceTest( + min_age=16, min_usual_weekly_hours=30 + ), + ), + } + ) + _, membership = build_benefit_units(person, household, working) + assert membership.tolist() == [1, 2] + + def test_parents_in_different_units_are_refused_or_resolved(self) -> None: + person, household = self._frame( + [ + (1, 1, 45, None, None, None, 0.0), + (2, 1, 44, None, None, None, 0.0), + (3, 1, 5, None, 2, 1, 0.0), + ], + {1: 1}, + ) + with pytest.raises(ValueError, match="different units"): + build_benefit_units(person, household, self.RULE) + first = BenefitUnitRule( + **{**self._fields(), "split_parents": SplitParentsPolicy.FIRST_PARENT_UNIT} + ) + _, membership = build_benefit_units(person, household, first) + assert membership.tolist() == [1, 2, 2] + + @pytest.mark.parametrize( + ("rows", "message"), + [ + ( + [(1, 1, 30, 2, None, None, 0.0), (2, 2, 30, 1, None, None, 0.0)], + "cross_household", + ), + ( + [(1, 1, 30, 2, None, None, 0.0), (2, 1, 30, None, None, None, 0.0)], + "asymmetric", + ), + ([(1, 1, 30, None, 9, None, 0.0)], "dangling"), + ], + ) + def test_invalid_pointers_are_refused(self, rows, message) -> None: + person, household = self._frame(rows, {1: 1, 2: 2}) + with pytest.raises(ValueError, match=message): + build_benefit_units(person, household, self.RULE) + + def test_missing_columns_are_named(self) -> None: + person, household = self._frame([(1, 1, 30, None, None, None, 0.0)], {1: 1}) + with pytest.raises(ValueError, match="parent_2_person_id"): + build_benefit_units( + person.drop(columns="parent_2_person_id"), household, self.RULE + ) + + def _fields(self) -> dict[str, object]: + rule = self.RULE + return { + "entity": rule.entity, + "dependent_child": rule.dependent_child, + "unparented_child": rule.unparented_child, + "split_parents": rule.split_parents, + "provenance": rule.provenance, + } + + +class TestRuleSerialization: + @PROPERTY + @given(rule=unit_rules(refusing=True)) + def test_rules_round_trip_through_json(self, rule) -> None: + payload = json.loads(json.dumps(rule.to_dict())) + assert payload["format"] == BENEFIT_UNIT_RULE_FORMAT + assert BenefitUnitRule.from_dict(payload) == rule + assert hash(BenefitUnitRule.from_dict(payload)) == hash(rule) + + @pytest.mark.parametrize( + "change", + [ + {"format": "x"}, + {"entity": "household"}, + {"entity": "Family"}, + {"unparented_child": "nowhere"}, + {"split_parents": 3}, + {"provenance": {"": "x"}}, + {"dependent_child": {"max_age": 17}}, + {"dependent_child": {"max_age": True, "financial_independence": None}}, + {"dependent_child": {"max_age": 200, "financial_independence": None}}, + { + "dependent_child": { + "max_age": 17, + "financial_independence": { + "min_age": 18, + "min_usual_weekly_hours": 30, + }, + } + }, + { + "dependent_child": { + "max_age": 17, + "financial_independence": { + "min_age": 16, + "min_usual_weekly_hours": 0, + }, + } + }, + {"extra": 1}, + ], + ) + def test_malformed_rules_raise_value_error(self, change) -> None: + payload = {**TestPlacement.RULE.to_dict(), **change} + with pytest.raises(ValueError): + BenefitUnitRule.from_dict(payload) + + def test_a_test_must_be_declared_as_one(self) -> None: + with pytest.raises(ValueError, match="declared as one"): + DependentChildRule( + max_age=17, + financial_independence={"min_age": 16, "min_usual_weekly_hours": 30}, + ) + + @PROPERTY + @given( + data=st.recursive( + st.none() | st.booleans() | st.integers() | st.text(max_size=5), + lambda inner: ( + st.lists(inner, max_size=3) + | st.dictionaries(st.text(max_size=5), inner, max_size=3) + ), + max_leaves=8, + ) + ) + def test_arbitrary_payloads_only_raise_value_error(self, data) -> None: + with pytest.raises(ValueError): + BenefitUnitRule.from_dict(data) diff --git a/packages/microcosm-frame/tests/golden/concept-coverage/axiom-nz.json b/packages/microcosm-frame/tests/golden/concept-coverage/axiom-nz.json index 074030417d..21e2be98e4 100644 --- a/packages/microcosm-frame/tests/golden/concept-coverage/axiom-nz.json +++ b/packages/microcosm-frame/tests/golden/concept-coverage/axiom-nz.json @@ -1,6 +1,40 @@ { "concept_inputs": { + "fact:household.mortgage_interest": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_owner_weekly_required_payments" + } + ], + "fact:household.mortgage_principal": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_owner_weekly_required_payments" + } + ], "fact:household.reference_person_id": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_dependent_children" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_two_or_more_dependent_children" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_owner_weekly_required_payments" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_sole_parent" + }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -25,6 +59,11 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_has_accommodation_costs" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_owns_premises_and_not_joint_owner_with_resident" + }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -42,6 +81,16 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_has_dependent_children" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_two_or_more_dependent_children" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_sole_parent" + }, { "entity": "Person", "module": "nz/statutes/social_security/orphans_unsupported_child_benefit/core.yaml", @@ -209,10 +258,35 @@ } ], "fact:person.dividend_income": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, { "entity": "Person", "module": "nz/statutes/income_tax/withholding/resident_withholding_tax.yaml", "name": "rwt_gross_interest_and_dividends" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" } ], "fact:person.education_enrollment": [ @@ -233,16 +307,31 @@ "module": "nz/regulations/acc/earners_levy.yaml", "name": "acc_employee_earnings_for_low_self_employed_formula" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, { "entity": "Person", "module": "nz/statutes/income_tax/core/taxable_income.yaml", "name": "income_tax_employment_income" }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, { "entity": "Person", "module": "nz/statutes/kiwisaver/contributions.yaml", "name": "kiwisaver_gross_salary_or_wages" }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, { "entity": "Person", "module": "nz/statutes/income_tax/credits/individual_credits.yaml", @@ -253,6 +342,11 @@ "module": "nz/statutes/parental_leave/paid_parental_leave.yaml", "name": "ppl_employees_average_weekly_earnings" }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, { "entity": "Person", "module": "nz/statutes/student_loan/repayments.yaml", @@ -272,6 +366,11 @@ "entity": "Person", "module": "nz/statutes/social_security/main_benefits/rates.yaml", "name": "supported_living_payment_beneficiary_personal_earnings" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" } ], "fact:person.enrolled_full_time": [ @@ -297,6 +396,11 @@ "module": "nz/regulations/acc/earners_levy.yaml", "name": "acc_self_employed_earnings_for_earners_levy" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, { "entity": "Family", "module": "nz/statutes/income_tax/family_scheme/family_scheme_income.yaml", @@ -306,6 +410,26 @@ "entity": "Person", "module": "nz/statutes/income_tax/core/taxable_income.yaml", "name": "income_tax_activity_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" } ], "fact:person.has_disability": [ @@ -321,10 +445,35 @@ } ], "fact:person.interest_income": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, { "entity": "Person", "module": "nz/statutes/income_tax/withholding/resident_withholding_tax.yaml", "name": "rwt_gross_interest_and_dividends" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" } ], "fact:person.legal_marital_status": [ @@ -350,6 +499,11 @@ "module": "nz/regulations/acc/earners_levy.yaml", "name": "acc_self_employed_earnings_for_earners_levy" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, { "entity": "Family", "module": "nz/statutes/income_tax/family_scheme/family_scheme_income.yaml", @@ -359,6 +513,26 @@ "entity": "Person", "module": "nz/statutes/income_tax/core/taxable_income.yaml", "name": "income_tax_activity_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" } ], "fact:person.parent_1_person_id": [ @@ -367,6 +541,16 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_has_dependent_children" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_two_or_more_dependent_children" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_sole_parent" + }, { "entity": "Person", "module": "nz/regulations/social_security/childcare_assistance/core.yaml", @@ -464,6 +648,16 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_has_dependent_children" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_two_or_more_dependent_children" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_sole_parent" + }, { "entity": "Person", "module": "nz/regulations/social_security/childcare_assistance/core.yaml", @@ -556,11 +750,26 @@ } ], "fact:person.partner_person_id": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_dependent_children" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_two_or_more_dependent_children" + }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_in_relationship" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_sole_parent" + }, { "entity": "Person", "module": "nz/regulations/health_entitlement_cards/community_services_card/core.yaml", @@ -627,6 +836,33 @@ "name": "young_person_is_in_relationship" } ], + "fact:person.private_pension_income": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" + } + ], "fact:person.public_pension_income": [ { "entity": "Person", @@ -649,6 +885,33 @@ "name": "winter_energy_payment_receiving_new_zealand_superannuation" } ], + "fact:person.rental_income": [ + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" + } + ], "fact:person.take_up_seed": [ { "entity": "Person", @@ -662,7 +925,7 @@ } ] }, - "covered_count": 79, + "covered_count": 88, "covered_inputs": [ { "entity": "Person", @@ -694,11 +957,31 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_has_dependent_children" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_has_two_or_more_dependent_children" + }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_in_relationship" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_owner_weekly_required_payments" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_owns_premises_and_not_joint_owner_with_resident" + }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_relevant_weekly_income" + }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -709,6 +992,11 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_rent_paid_to_registered_community_housing_provider_social_housing" }, + { + "entity": "Family", + "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", + "name": "accommodation_supplement_sole_parent" + }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -839,11 +1127,21 @@ "module": "nz/statutes/social_security/main_benefits/rates.yaml", "name": "jobseeker_support_single" }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "jobseeker_support_total_income" + }, { "entity": "Person", "module": "nz/statutes/kiwisaver/contributions.yaml", "name": "kiwisaver_gross_salary_or_wages" }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "main_benefit_income_test_assessable_income" + }, { "entity": "Person", "module": "nz/statutes/social_security/main_benefits/entitlement.yaml", @@ -924,6 +1222,11 @@ "module": "nz/statutes/social_security/main_benefits/entitlement.yaml", "name": "sole_parent_spouse_or_partner_died" }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "sole_parent_support_total_income" + }, { "entity": "Person", "module": "nz/regulations/student_allowances/core.yaml", @@ -999,6 +1302,11 @@ "module": "nz/statutes/social_security/main_benefits/rates.yaml", "name": "supported_living_payment_single" }, + { + "entity": "Person", + "module": "nz/statutes/social_security/main_benefits/rates.yaml", + "name": "supported_living_payment_total_income" + }, { "entity": "Family", "module": "nz/statutes/income_tax/family_scheme/eligibility.yaml", @@ -1196,11 +1504,6 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_disability_accommodation_or_care_funded_under_pae_ora_act" }, - { - "entity": "Family", - "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", - "name": "accommodation_supplement_has_two_or_more_dependent_children" - }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -1221,16 +1524,6 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_owner_weekly_payment_share" }, - { - "entity": "Family", - "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", - "name": "accommodation_supplement_owner_weekly_required_payments" - }, - { - "entity": "Family", - "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", - "name": "accommodation_supplement_owns_premises_and_not_joint_owner_with_resident" - }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -1241,11 +1534,6 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_receiving_or_eligible_for_student_allowance_grant" }, - { - "entity": "Family", - "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", - "name": "accommodation_supplement_relevant_weekly_income" - }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -1281,11 +1569,6 @@ "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", "name": "accommodation_supplement_social_housing_weekly_contributions_paid" }, - { - "entity": "Family", - "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", - "name": "accommodation_supplement_sole_parent" - }, { "entity": "Family", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", @@ -2996,11 +3279,6 @@ "module": "nz/statutes/social_security/main_benefits/rates.yaml", "name": "jobseeker_support_partner_receives_new_zealand_superannuation_or_veterans_pension" }, - { - "entity": "Person", - "module": "nz/statutes/social_security/main_benefits/rates.yaml", - "name": "jobseeker_support_total_income" - }, { "entity": "Person", "module": "nz/statutes/social_security/main_benefits/rates.yaml", @@ -3061,11 +3339,6 @@ "module": "nz/statutes/child_support/core_formula.yaml", "name": "living_allowance" }, - { - "entity": "Person", - "module": "nz/statutes/social_security/main_benefits/rates.yaml", - "name": "main_benefit_income_test_assessable_income" - }, { "entity": "Person", "module": "nz/statutes/social_security/main_benefits/entitlement.yaml", @@ -3531,11 +3804,6 @@ "module": "nz/statutes/social_security/main_benefits/rates.yaml", "name": "sole_parent_support_personal_earnings_used_for_childcare" }, - { - "entity": "Person", - "module": "nz/statutes/social_security/main_benefits/rates.yaml", - "name": "sole_parent_support_total_income" - }, { "entity": "Person", "module": "nz/statutes/social_security/main_benefits/entitlement.yaml", @@ -3681,11 +3949,6 @@ "module": "nz/statutes/social_security/main_benefits/rates.yaml", "name": "supported_living_payment_restricted_or_blind" }, - { - "entity": "Person", - "module": "nz/statutes/social_security/main_benefits/rates.yaml", - "name": "supported_living_payment_total_income" - }, { "entity": "Person", "module": "nz/statutes/social_security/main_benefits/rates.yaml", @@ -4014,13 +4277,9 @@ ], "unknown_inputs": [], "unmapped_concepts": { - "fact:household.mortgage_interest": "rulespec-nz at 6fe181f has no input mortgage interest determines on its own: the only owner-cost input, accommodation_supplement_owner_weekly_required_payments (Family, social_security/accommodation_supplement/core.yaml, s 65AAA), is the owner's weekly 'required mortgage-security payments and reasonably required home payments', which needs interest and principal (and other home payments) summed and then scaled by 1/52, and no transform composes a sum with a scale; accommodation_supplement_concessionary_mortgage_payments_to_kainga_ora_or_crown (s 66) needs the lender's identity.", - "fact:household.mortgage_principal": "rulespec-nz at 6fe181f has no input mortgage principal determines on its own: the only owner-cost input, accommodation_supplement_owner_weekly_required_payments (Family, social_security/accommodation_supplement/core.yaml, s 65AAA), needs principal and interest (and other home payments) summed and then scaled to a weekly amount, and no transform composes a sum with a scale.", "fact:household.property_tax": "The only rates input, rr_rates_payable_before_rebate (Person) in statutes/rates_rebates/rates_rebate.yaml, cannot be bound at 6fe181f, because the dense engine could not compile that module and nz.json lists it only under cli_walk_inputs, which the adapter's surface loader does not read; once the module compiles, allocate_to_reference_person of property_tax is the candidate (approximate, because the ratepayer need not be the reference person).", "fact:person.educational_attainment": "rulespec-nz at 6fe181f has no input for completed education: no rule or input name mentions a qualification, degree, diploma or attainment level; the education inputs are current enrollment or school attendance (this mapping binds some of them to education_enrollment and enrolled_full_time); and young_person_is_study_ready is what youth_young_parent_payment/core.yaml calls an agency-adjudicated composite input (s 50).", - "fact:person.private_pension_income": "rulespec-nz at 6fe181f has no input the concept determines: the pension inputs are Family-level family-scheme items defined by contribution source or exempt status rather than receipt (family_scheme_superannuation_distribution_income under MB 5 counts only distributions attributable to employer contributions; family_scheme_retirement_scheme_distribution_income is MB 6; family_scheme_exempt_pension_or_annuity_amount is MB 10's exempt pensions and annuities, half included), and community_services_card_raw_superannuation_income is the reg 8(5A) superannuitant income-test total.", "fact:person.realized_capital_gains": "rulespec-nz at 6fe181f has no input for gains on disposals: no rule mentions gains, disposals or the bright-line test, and the only capital input, income_tax_expenditure_or_loss_is_capital_nature (income_tax/core/taxable_income.yaml, s DA 2), classifies expenditure for the general deduction limitation.", - "fact:person.rental_income": "rulespec-nz at 6fe181f has no input for rental or other property income: the only activity-income inputs (income_tax_activity_income in income_tax/core/taxable_income.yaml and family_scheme_activity_income in family_scheme_income.yaml) are the MB 3 bridge the module labels 'business and self-employment activity income' and are already bound to self-employment profit; accommodation_supplement_weekly_rent_received_from_other_residents is rent from co-residents of the own dwelling, which the concept excludes; and the benefit income-test inputs are weekly totals of all income.", "fact:person.sex": "rulespec-nz at 6fe181f has no sex- or gender-dependent input: none of the 668 dense-surface input names or 40 module YAMLs mention sex, gender, male, female, man, woman, wife or husband, and the only related hit, sole_parent_is_mother_or_father_or_child_regarded_as_child (social_security/main_benefits/entitlement.yaml), tests parenthood, which this mapping binds to the parent_of_coresident_child role.", "fact:person.usual_weekly_hours": "rulespec-nz at 6fe181f has no input that takes hours worked: the work-hours inputs are threshold statuses no transform can compute from hours, namely acc_self_employed_more_than_30_hours_per_week (regulations/acc/earners_levy.yaml, reg 6, which also needs self-employment hours alone), jobseeker_in_full_time_employment (main_benefits/entitlement.yaml, s 21), acc_weekly_compensation_is_full_time_earner and the Family-level in_work_tax_credit_normally_full_time_earner; the remaining hours_per_week inputs are childcare, programme-participation or work-capacity hours.", "fact:person.weeks_worked": "rulespec-nz at 6fe181f has no input the concept determines: the closest, in_work_tax_credit_weekly_periods (Family, income_tax/family_scheme/tax_credits.yaml, s MD 10(2)), counts weeks in which the person or their partner meets the earner requirement, which needs the union of two partners' work weeks that no group rule gives (reference_member drops a working partner; sum_over_members double-counts and adds children's weeks), and minimum_family_tax_credit_weekly_periods and minimum_family_full_time_earner_weeks count only weeks as a full-time earner." diff --git a/test_support/microcosm_frame/concept_engine_frames.py b/test_support/microcosm_frame/concept_engine_frames.py index 6037e21879..3a3022f555 100644 --- a/test_support/microcosm_frame/concept_engine_frames.py +++ b/test_support/microcosm_frame/concept_engine_frames.py @@ -32,6 +32,7 @@ Recode, RelationshipRole, Scale, + ScaledSum, Share, Sum, TakeUpThreshold, @@ -166,7 +167,13 @@ def expected_input_dtypes(binding: InputBinding) -> set[str]: return {"int"} if isinstance( transform, - Share | Fraction | Scale | Sum | Product | AllocateToReferencePerson, + Share + | Fraction + | Scale + | Sum + | ScaledSum + | Product + | AllocateToReferencePerson, ): return {"float"} if isinstance(transform, Recode): diff --git a/test_support/microcosm_frame/concept_reference.py b/test_support/microcosm_frame/concept_reference.py index 7eac5c8fb3..aaeec57900 100644 --- a/test_support/microcosm_frame/concept_reference.py +++ b/test_support/microcosm_frame/concept_reference.py @@ -31,6 +31,7 @@ RelationshipRole, Role, Scale, + ScaledSum, Share, Sum, TakeUpThreshold, @@ -101,6 +102,9 @@ def is_parent_of(row: dict, child: dict, max_age: int | None) -> bool: out.append(float(value(row, concepts[0])) > 0) elif isinstance(transform, Sum): out.append(sum(float(value(row, concept_id)) for concept_id in concepts)) + elif isinstance(transform, ScaledSum): + total = sum(float(value(row, concept_id)) for concept_id in concepts) + out.append(total * transform.factor) elif isinstance(transform, Product): first, second = (float(value(row, concept_id)) for concept_id in concepts) out.append(first * second) diff --git a/test_support/microcosm_frame/unit_rules.py b/test_support/microcosm_frame/unit_rules.py new file mode 100644 index 0000000000..f09861c126 --- /dev/null +++ b/test_support/microcosm_frame/unit_rules.py @@ -0,0 +1,47 @@ +"""Hypothesis strategies for benefit-unit rules, shared across test groups.""" + +# ruff: noqa: F401 + +from __future__ import annotations + +from hypothesis import strategies as st + +from microcosm.frame.unit_construction import ( + BenefitUnitRule, + DependentChildRule, + FinancialIndependenceTest, + SplitParentsPolicy, + UnparentedChildPlacement, +) + + +@st.composite +def unit_rules(draw, *, refusing: bool = False) -> BenefitUnitRule: + """A unit rule; non-refusing unless ``refusing``, so any valid frame builds.""" + + max_age = draw(st.integers(0, 30)) + test = None + if draw(st.booleans()): + test = FinancialIndependenceTest( + min_age=draw(st.integers(0, max_age)), + min_usual_weekly_hours=draw( + st.floats(min_value=0.5, max_value=168.0, allow_nan=False) + ), + ) + placements = list(UnparentedChildPlacement) + splits = list(SplitParentsPolicy) + if not refusing: + placements.remove(UnparentedChildPlacement.REFUSE) + splits.remove(SplitParentsPolicy.REFUSE) + return BenefitUnitRule( + entity=draw(st.sampled_from(["family", "benefit_unit"])), + dependent_child=DependentChildRule( + max_age=max_age, financial_independence=test + ), + unparented_child=draw(st.sampled_from(placements)), + split_parents=draw(st.sampled_from(splits)), + provenance={"max_age": "generated"}, + ) + + +__all__ = [name for name in globals() if not name.startswith("__")] From d6a7c2dd3a49e5330e00a53cd23e0cf69b7db444 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 07:09:32 -0400 Subject: [PATCH 2/5] Fix issues from review: unit roles with negative ids, pin the NZ mapping (G4) Applies the independent review of #1122 (APPROVE_WITH_NITS at 11521ae6c). F1. benefit_unit_roles marked "no partner" with -1 and compared that marker with person ids, so a person whose id is -1 was read as the partner of a head who has none (a sole parent and child -1 came out as a couple with no children). Partner presence is now a mask and the head is found by row position, so no id can pass for a missing partner. The concept-frame contract accepts any unique integer id. - Regression: TestPlacement.test_a_person_id_of_minus_one_is_a_person_not_a_missing_partner (and negative ids in every role). - concept_frames takes min_id; when it is negative it also mixes in ids near zero, since a draw from +/-10**9 never hit -1 in 300 examples. The unit-construction, group-encode and NZ closure properties now draw negative ids (default draws for every other caller are unchanged). - New unit-construction invariant: the partner role is held by the head's partner and nobody else. On the old code it and eight group-encode properties fail. F2. The membership check that a partner is its head's partner had no test of its own (mutant M13, `if False:`, survived). New refusal case: a child in a sole parent's unit relabelled "partner". The swapped-roles case now matches that check's message rather than any message containing "partner". F3. The new axiom:nz mapping content is pinned against hand-computed households (a couple with two children, wages and a mortgage plus an adult boarder; a renting sole parent with one child; every income concept non-zero somewhere), with weekly values worked by hand from annual amounts, plus a check that every NZ Scale and ScaledSum reads annual-flow concepts, says 1/52 in its note and divides by 52. Mutants M28, M29, M30, M31 and M34 pass the PR's original tests and fail these. Twelve sibling mutants (feature swaps, other factors, group rules, dropped concepts) also fail the new tests: ten on their assertions, two already at mapping load. F4. The changelog said the closure closes all 140 root inputs; it closes 139, and the cash-asset input awaits G3a. F6. The closure test cited build/nz benefit_unit_rule.json, which G2 (#1119) adds; the comment now says so. The ADR's units invariant now states the partner-role rule and that it holds for negative ids. Not changed here: F5 (two dependent-child definitions in axiom:nz, a D1 methodology question) and F7 (the engine-present golden-08 run, which needs an axiom_rules_engine build). axiom: n/a: microsim-side unit construction and tests; no policy is encoded in a country model. Co-Authored-By: Claude Opus 5.5 --- .../unit-construction-group-encode.added.md | 2 +- docs/concept-schema-transport-adr.md | 10 +- .../shared/test_nz_axiom_input_closure.py | 10 +- .../src/microcosm/frame/unit_construction.py | 10 +- .../engine_free/shared/test_group_encode.py | 222 +++++++++++++++++- .../shared/test_unit_construction.py | 62 ++++- .../microcosm_frame/concept_frames.py | 19 +- 7 files changed, 308 insertions(+), 27 deletions(-) diff --git a/changelog.d/unit-construction-group-encode.added.md b/changelog.d/unit-construction-group-encode.added.md index a26a2ba730..4cf6f2bf81 100644 --- a/changelog.d/unit-construction-group-encode.added.md +++ b/changelog.d/unit-construction-group-encode.added.md @@ -1 +1 @@ -Add benefit-unit construction from concept pointers (`microcosm.frame.unit_construction`), group encoding that executes a mapping's group rules, state bindings and scenario knobs on built units (`ConceptMapping.encode_groups`), and Axiom input closures (`microcosm.frame.input_closure`). Extend the `axiom:nz` mapping with the Accommodation Supplement unit-composition flags, owner payments and relevant income, and the main-benefit rates income inputs, and commit the New Zealand input closure (`build/nz/axiom_input_closure.json`), which closes all 140 root inputs of the four bound modules against the committed input surface. +Add benefit-unit construction from concept pointers (`microcosm.frame.unit_construction`), group encoding that executes a mapping's group rules, state bindings and scenario knobs on built units (`ConceptMapping.encode_groups`), and Axiom input closures (`microcosm.frame.input_closure`). Extend the `axiom:nz` mapping with the Accommodation Supplement unit-composition flags, owner payments and relevant income, and the main-benefit rates income inputs, and commit the New Zealand input closure (`build/nz/axiom_input_closure.json`), which closes 139 of the 140 root inputs of the four bound modules against the committed input surface; the 140th, the Accommodation Supplement cash-asset input, awaits the liquid-asset binding (G3a). diff --git a/docs/concept-schema-transport-adr.md b/docs/concept-schema-transport-adr.md index dd93c9858a..f6f039525b 100644 --- a/docs/concept-schema-transport-adr.md +++ b/docs/concept-schema-transport-adr.md @@ -294,10 +294,12 @@ These hold for every input and are tested (Hypothesis properties unless noted): closures. 11. **Units.** Benefit units partition persons: every person is in exactly one unit, every unit has a head and at most one partner, partners share a - unit, every dependent child shares a unit with a co-resident parent (or, - with none, with the household reference person), and units nest in - households. Units carry no weights; each takes its household's, so the - sum of unit weights is the sum over households of household weight times + unit and the partner role goes to the head's partner alone, every + dependent child shares a unit with a co-resident parent (or, with none, + with the household reference person), and units nest in households, + whatever unique integer ids the persons carry, negative ones included. + Units carry no weights; each takes its household's, so the sum of unit + weights is the sum over households of household weight times units in the household. Unit ids are deterministic and independent of row order. Group encoding agrees with a row-by-row reference, and its unit-composition flags agree with the unit attributes computed diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py index af1df00d79..e39660a906 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py @@ -66,7 +66,8 @@ #: closure check reports the entry until its awaiting note is removed. AWAITING = {entry.ref for entry in CLOSURE.entries if entry.awaiting is not None} #: Stand-in unit rule for these tests. The NZ rule is spec data (method card -#: MC7, build/nz benefit_unit_rule.json); these properties hold for any rule. +#: MC7, build/nz/benefit_unit_rule.json, which G2 adds); these properties +#: hold for any rule. RULE = BenefitUnitRule( entity="family", dependent_child=DependentChildRule(max_age=17, financial_independence=None), @@ -236,7 +237,10 @@ def _family_inputs(tables, knobs=None) -> pd.DataFrame: class TestExecutableClosure: @settings(max_examples=60, deadline=None) - @given(tables=concept_frames(max_households=4, max_members=6), data=st.data()) + @given( + tables=concept_frames(max_households=4, max_members=6, min_id=-(10**9)), + data=st.data(), + ) def test_encoding_and_defaults_supply_every_supplement_input( self, tables, data ) -> None: @@ -261,7 +265,7 @@ def test_encoding_and_defaults_supply_every_supplement_input( assert family[name].dtype.kind in "bif", name @settings(max_examples=40, deadline=None) - @given(tables=concept_frames(max_households=4, max_members=6)) + @given(tables=concept_frames(max_households=4, max_members=6, min_id=-(10**9))) def test_encoding_and_defaults_supply_every_person_input(self, tables) -> None: # The v0 frame is transported: donor program receipts are dropped, so # a binding that reads one never runs and the closure feeds its input. diff --git a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py index d6876768a0..535f96bda4 100644 --- a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py +++ b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py @@ -404,11 +404,15 @@ def benefit_unit_roles( units = _unit_rows(family, membership, rule) head_of = family[rule.head_column].to_numpy(dtype=np.int64)[units] is_head = ids == head_of + # Partner presence is a mask, not a sentinel id: any integer, -1 + # included, can be a person id. present = person["partner_person_id"].notna().to_numpy() - partner = np.full(len(ids), -1, dtype=np.int64) + partner = np.zeros(len(ids), dtype=np.int64) partner[present] = person["partner_person_id"][present].to_numpy(dtype=np.int64) - head_partner = pd.Series(partner, index=ids).reindex(head_of).to_numpy() - is_partner = ~is_head & (head_partner == ids) + head_row = pd.Index(ids).get_indexer(head_of) + found = head_row >= 0 + head_row = np.where(found, head_row, 0) + is_partner = ~is_head & found & present[head_row] & (partner[head_row] == ids) roles = np.where( is_head, UnitRole.HEAD.value, diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py index 5118e2e67b..420ec55818 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py @@ -50,7 +50,7 @@ transform_from_dict, transform_to_dict, ) -from microcosm.frame.concepts import CONCEPTS, AlignmentRelation +from microcosm.frame.concepts import CONCEPTS, AlignmentRelation, TemporalBasis from microcosm.frame.unit_construction import ( BenefitUnitRule, DependentChildRule, @@ -202,7 +202,7 @@ def _mapping(bindings=BINDINGS) -> ConceptMapping: def unit_frames(draw): """A concept frame with model state, its units, and its membership.""" - tables = draw(concept_frames(max_households=5, max_members=6)) + tables = draw(concept_frames(max_households=5, max_members=6, min_id=-(10**9))) rule = draw(unit_rules()) person = tables["person"].copy() household = tables["household"].copy() @@ -775,7 +775,7 @@ def test_roles_must_agree_with_the_partner_pointers(self) -> None: bad = GroupMembership( entity="family", units=family, person_unit=membership, person_role=swapped ) - with pytest.raises(ValueError, match="partner"): + with pytest.raises(ValueError, match="partner must be its head's partner"): MAPPING.encode_groups(tables, {"Family": bad}) demoted = roles.copy() demoted.iloc[1] = UnitRole.DEPENDENT_CHILD.value @@ -785,6 +785,34 @@ def test_roles_must_agree_with_the_partner_pointers(self) -> None: with pytest.raises(ValueError, match="must be its partner"): MAPPING.encode_groups(tables, {"Family": bad}) + def test_a_child_relabelled_as_a_partner_is_refused(self) -> None: + tables, rule, _, _, _ = _example() + # Give the lone adult 13 a child, 14, so 13 heads a unit with no + # partner in it: no head-side check can see a partner label there. + person = tables["person"] + child = person.iloc[[2]].assign(person_id=14) + child["parent_1_person_id"] = pd.array([13], dtype="Int64") + person = pd.concat([person, child], ignore_index=True) + tables = {**tables, "person": person} + family, membership = build_benefit_units(person, tables["household"], rule) + group = benefit_unit_membership(person, family, membership, rule) + roles = group.person_role.copy() + assert membership.tolist() == [10, 10, 10, 13, 13] + assert roles.tolist() == [ + "head", + "partner", + "dependent_child", + "head", + "dependent_child", + ] + MAPPING.encode_groups(tables, {"Family": group}) + roles.iloc[4] = UnitRole.PARTNER.value + bad = GroupMembership( + entity="family", units=family, person_unit=membership, person_role=roles + ) + with pytest.raises(ValueError, match="partner must be its head's partner"): + MAPPING.encode_groups(tables, {"Family": bad}) + def test_memberships_are_typed_and_distinct(self) -> None: tables, _, _, _, group = _example() with pytest.raises(ValueError, match="not a GroupMembership"): @@ -976,7 +1004,7 @@ class TestNewZealandMapping: ) @PROPERTY - @given(tables=concept_frames(max_households=4, max_members=6)) + @given(tables=concept_frames(max_households=4, max_members=6, min_id=-(10**9))) def test_every_accommodation_supplement_group_binding_executes( self, tables ) -> None: @@ -1000,3 +1028,189 @@ def test_every_accommodation_supplement_group_binding_executes( assert kind in ("b", "f"), binding.engine_input rent = table["accommodation_supplement_weekly_rent_paid"].sum() np.testing.assert_allclose(rent, household["rent"].sum() * WEEK, rtol=1e-12) + + # Hand-computed households. Concept amounts are annual flows and these + # inputs are weekly amounts, so every expected value below is the annual + # figure divided by 52 weeks, worked by hand rather than read from a + # binding's factor. + RATES = "nz/statutes/social_security/main_benefits/rates.yaml" + #: Each person's own weekly income from the seven income concepts. + OWN_WEEKLY_INCOME = { + 101: 1_000.0 + 10.0, # wages 52,000; interest 520 + 102: 200.0 + 20.0, # non-farm self-employment 10,400; dividends 1,040 + 103: 0.0, + 104: 0.0, + 105: 400.0 + 50.0 + 50.0, # wages 20,800; rent 2,600; private pension 2,600 + 201: 250.0 + 50.0, # wages 13,000; farm self-employment 2,600 + 202: 0.0, + } + + @staticmethod + def _households() -> dict[str, pd.DataFrame]: + """Two households, every income concept non-zero somewhere. + + Household 1 owns with a mortgage (interest 15,600 and principal 5,200 + a year): a couple, 101 (the reference person) and 102, their children + 103 and 104, and an adult boarder, 105. Household 2 rents privately at + 15,600 a year: a sole parent, 201, and their child, 202. + """ + + rows = [ + (101, 1, 41, 102, None, None), + (102, 1, 39, 101, None, None), + (103, 1, 9, None, 101, 102), + (104, 1, 6, None, 101, 102), + (105, 1, 30, None, None, None), + (201, 2, 28, None, None, None), + (202, 2, 3, None, 201, None), + ] + person = pd.DataFrame( + rows, + columns=[ + "person_id", + "person_household_id", + "age", + "partner_person_id", + "parent_1_person_id", + "parent_2_person_id", + ], + ) + for column in ("partner_person_id", "parent_1_person_id", "parent_2_person_id"): + person[column] = pd.array(person[column].tolist(), dtype="Int64") + for item in CONCEPTS: + if item.entity == "person" and item.monetary is not None: + person[item.name] = 0.0 + annual = { + "employment_income": {101: 52_000.0, 105: 20_800.0, 201: 13_000.0}, + "interest_income": {101: 520.0}, + "nonfarm_self_employment_income": {102: 10_400.0}, + "dividend_income": {102: 1_040.0}, + "rental_income": {105: 2_600.0}, + "private_pension_income": {105: 2_600.0}, + "farm_self_employment_income": {201: 2_600.0}, + } + for column, amounts in annual.items(): + person[column] = person["person_id"].map(amounts).fillna(0.0) + household = pd.DataFrame( + { + "household_id": [1, 2], + "reference_person_id": pd.array([101, 201], dtype="Int64"), + "tenure": ["owned_with_mortgage", "rented_private"], + } + ) + for item in CONCEPTS: + if item.entity == "household" and item.monetary is not None: + household[item.name] = 0.0 + household["mortgage_interest"] = [15_600.0, 0.0] + household["mortgage_principal"] = [5_200.0, 0.0] + household["rent"] = [0.0, 15_600.0] + return {"person": person, "household": household} + + def test_hand_computed_households_get_the_supplement_inputs(self) -> None: + tables = self._households() + person, household = tables["person"], tables["household"] + family, membership = build_benefit_units(person, household, self.NZ_RULE) + group = benefit_unit_membership(person, family, membership, self.NZ_RULE) + table = ( + axiom_concept_mapping("nz") + .encode_groups(tables, {"Family": group}, modules=[self.AS]) + .tables["family"] + ) + # The couple's family (101), the boarder's (105), the sole parent's (201). + assert table["family_id"].tolist() == [101, 105, 201] + flags = { + "accommodation_supplement_in_relationship": [True, False, False], + "accommodation_supplement_has_dependent_children": [True, False, True], + "accommodation_supplement_has_two_or_more_dependent_children": [ + True, + False, + False, + ], + "accommodation_supplement_sole_parent": [False, False, True], + # Every family in an owner-occupied household, the boarder's too. + "accommodation_supplement_owns_premises_and_not_joint_owner_with_resident": [ + True, + True, + False, + ], + } + for name, expected in flags.items(): + assert table[name].dtype == bool, name + assert table[name].tolist() == expected, name + income = self.OWN_WEEKLY_INCOME + amounts = { + # (15,600 + 5,200) / 52, on the family holding the reference person. + "accommodation_supplement_owner_weekly_required_payments": [400.0, 0, 0], + # 15,600 / 52. + "accommodation_supplement_weekly_rent_paid": [0, 0, 300.0], + # Summed over each family's members. + "accommodation_supplement_relevant_weekly_income": [ + income[101] + income[102] + income[103] + income[104], + income[105], + income[201] + income[202], + ], + } + assert amounts["accommodation_supplement_relevant_weekly_income"] == [ + 1_230.0, + 500.0, + 300.0, + ] + for name, expected in amounts.items(): + assert table[name].dtype.kind == "f", name + np.testing.assert_allclose( + table[name], expected, rtol=1e-12, atol=1e-9, err_msg=name + ) + + def test_main_benefit_income_inputs_are_weekly_own_income(self) -> None: + tables = self._households() + mapping = axiom_concept_mapping("nz") + encoded = mapping.encode( + tables, + shares={name: 0.5 for name in mapping.share_parameters()}, + take_up_rates={name: None for name in mapping.take_up_programs()}, + ).tables["person"] + names = { + "jobseeker_support_total_income", + "main_benefit_income_test_assessable_income", + "sole_parent_support_total_income", + "supported_living_payment_total_income", + } + assert names <= { + binding.engine_input + for binding in mapping.bindings + if binding.module == self.RATES + } + expected = [self.OWN_WEEKLY_INCOME[pid] for pid in encoded["person_id"]] + for name in sorted(names): + np.testing.assert_allclose( + encoded[name], expected, rtol=1e-12, atol=1e-9, err_msg=name + ) + + def test_every_amount_conversion_is_annual_to_weekly(self) -> None: + # The New Zealand rules this mapping feeds state amounts weekly, and + # every amount concept is an annual flow, so each scaling divides by + # the 52 weeks of a year. A monthly or fortnightly input would need a + # different factor and a note saying so. + by_id = {item.id: item for item in CONCEPTS} + scaled = [ + binding + for binding in axiom_concept_mapping("nz").bindings + if isinstance(binding.transform, Scale | ScaledSum) + ] + assert {binding.engine_input for binding in scaled} >= { + "accommodation_supplement_owner_weekly_required_payments", + "accommodation_supplement_relevant_weekly_income", + "jobseeker_support_total_income", + "main_benefit_income_test_assessable_income", + "sole_parent_support_total_income", + "supported_living_payment_total_income", + } + for binding in scaled: + for concept_id in binding.concepts: + concept = by_id[concept_id] + assert concept.period == "year", (binding.engine_input, concept_id) + assert concept.temporal_basis is TemporalBasis.ANNUAL_FLOW + assert "1/52" in binding.note, binding.engine_input + assert binding.transform.factor * 52 == pytest.approx(1.0, abs=1e-15), ( + binding.engine_input + ) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py index 8fbc60449c..fc8142bb8c 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -7,7 +7,7 @@ - partition: every person is in exactly one unit, and every unit has members; - every unit has an adult (its head) and at most one partner; -- partners share a unit; +- partners share a unit, and the partner role is exactly the head's partner; - every dependent child shares a unit with a co-resident parent, or with the household's reference person when it has none; - units nest in households; @@ -43,6 +43,9 @@ from test_support.microcosm_frame.unit_rules import unit_rules PROPERTY = settings(max_examples=150, deadline=None) +#: Ids may be negative: the concept-frame contract accepts any unique integer, +#: so no unit reading may treat an id such as -1 as "absent". +FRAMES = concept_frames(max_households=5, max_members=6, min_id=-(10**9)) def _rows(person: pd.DataFrame, column: str) -> np.ndarray: @@ -65,7 +68,7 @@ def _reference_rows(person: pd.DataFrame, household: pd.DataFrame) -> np.ndarray class TestInvariants: @PROPERTY - @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + @given(tables=FRAMES, rule=unit_rules()) def test_units_partition_persons_and_nest_in_households(self, tables, rule) -> None: person, household = tables["person"], tables["household"] family, membership = build_benefit_units(person, household, rule) @@ -91,7 +94,7 @@ def test_units_partition_persons_and_nest_in_households(self, tables, rule) -> N assert membership.dtype == np.int64 @PROPERTY - @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + @given(tables=FRAMES, rule=unit_rules()) def test_every_unit_has_an_adult_and_partners_share_it(self, tables, rule) -> None: person, household = tables["person"], tables["household"] family, membership = build_benefit_units(person, household, rule) @@ -115,9 +118,18 @@ def test_every_unit_has_an_adult_and_partners_share_it(self, tables, rule) -> No membership.to_numpy()[paired], membership.to_numpy()[partner[paired]] ) assert not roles[paired].eq(UnitRole.DEPENDENT_CHILD.value).any() + # the partner role is held by the head's partner and nobody else + # (row positions, so no person id can pass for a missing partner) + head_row = pd.Index(person["person_id"].to_numpy()).get_indexer( + heads.reindex(membership.to_numpy()).to_numpy() + ) + assert np.array_equal( + roles.eq(UnitRole.PARTNER.value).to_numpy(), + partner[head_row] == np.arange(len(person)), + ) @PROPERTY - @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + @given(tables=FRAMES, rule=unit_rules()) def test_dependent_children_live_with_a_parent_or_caregiver( self, tables, rule ) -> None: @@ -152,7 +164,7 @@ def test_dependent_children_live_with_a_parent_or_caregiver( @PROPERTY @given( - tables=concept_frames(max_households=5, max_members=6), + tables=FRAMES, rule=unit_rules(), weights=st.lists( st.floats(min_value=0.001, max_value=1e6, allow_nan=False), @@ -198,7 +210,7 @@ def test_family_weights_are_inherited_from_households( @PROPERTY @given( - tables=concept_frames(max_households=5, max_members=6), + tables=FRAMES, rule=unit_rules(), data=st.data(), ) @@ -216,7 +228,7 @@ def test_units_do_not_depend_on_row_order(self, tables, rule, data) -> None: pd.testing.assert_series_equal(first.sort_index(), second.sort_index()) @PROPERTY - @given(tables=concept_frames(max_households=5, max_members=6), rule=unit_rules()) + @given(tables=FRAMES, rule=unit_rules()) def test_attributes_count_the_roles(self, tables, rule) -> None: person, household = tables["person"], tables["household"] family, membership = build_benefit_units(person, household, rule) @@ -381,6 +393,42 @@ def test_parents_in_different_units_are_refused_or_resolved(self) -> None: _, membership = build_benefit_units(person, household, first) assert membership.tolist() == [1, 2, 2] + def test_a_person_id_of_minus_one_is_a_person_not_a_missing_partner( + self, + ) -> None: + # A sole parent and their child, whose id is -1: the child is not the + # partner of a head who has none. + person, household = self._frame( + [(5, 1, 40, None, None, None, 0.0), (-1, 1, 10, None, 5, None, 0.0)], + {1: 5}, + ) + family, membership = build_benefit_units(person, household, self.RULE) + assert membership.tolist() == [5, 5] + roles = benefit_unit_roles(person, family, membership, self.RULE) + assert roles.tolist() == ["head", "dependent_child"] + attributes = benefit_unit_attributes(person, family, membership, self.RULE) + assert attributes["family_type"].tolist() == ["sole_parent"] + assert attributes["n_dependent_children"].tolist() == [1] + assert attributes["is_couple"].tolist() == [False] + # Negative ids in every role: a couple -1 and -2 with their child -3, + # and a lone adult -4 with no partner. + person, household = self._frame( + [ + (-1, 1, 40, -2, None, None, 0.0), + (-2, 1, 38, -1, None, None, 0.0), + (-3, 1, 10, None, -1, -2, 0.0), + (-4, 1, 30, None, None, None, 0.0), + ], + {1: -2}, + ) + family, membership = build_benefit_units(person, household, self.RULE) + assert family["family_id"].tolist() == [-4, -2] + assert membership.tolist() == [-2, -2, -2, -4] + roles = benefit_unit_roles(person, family, membership, self.RULE) + assert roles.tolist() == ["partner", "head", "dependent_child", "head"] + attributes = benefit_unit_attributes(person, family, membership, self.RULE) + assert attributes["family_type"].tolist() == ["single", "couple_with_children"] + @pytest.mark.parametrize( ("rows", "message"), [ diff --git a/test_support/microcosm_frame/concept_frames.py b/test_support/microcosm_frame/concept_frames.py index c8fcb09633..016b1bf62c 100644 --- a/test_support/microcosm_frame/concept_frames.py +++ b/test_support/microcosm_frame/concept_frames.py @@ -55,20 +55,29 @@ def concept_frames( min_households: int = 1, max_households: int = 4, max_members: int = 5, + min_id: int = 0, ) -> dict[str, pd.DataFrame]: - """A valid concept frame with every concept column present.""" + """A valid concept frame with every concept column present. + Person and household ids are drawn from ``[min_id, 10**9]``. The + concept-frame contract accepts any unique integer id, so a negative + ``min_id`` checks that no consumer reads an id as a sentinel such as -1; + ids near zero are then mixed in, because a draw from a wide range almost + never yields one exactly. + """ + + ids = st.integers(min_id, 10**9) + if min_id < 0: + ids |= st.integers(max(min_id, -3), 3) n_households = draw(st.integers(min_households, max_households)) sizes = [draw(st.integers(1, max_members)) for _ in range(n_households)] n_persons = sum(sizes) person_ids = draw( - st.lists( - st.integers(0, 10**9), min_size=n_persons, max_size=n_persons, unique=True - ) + st.lists(ids, min_size=n_persons, max_size=n_persons, unique=True) ) household_ids = draw( st.lists( - st.integers(0, 10**9), + ids, min_size=n_households, max_size=n_households, unique=True, From f1024b05e08bc1457c310697498b5c7af0bc949d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 14:10:25 -0400 Subject: [PATCH 3/5] Fix the G2/G3a merge into G4: foreign engine record, bound cash assets Merging main (G2 #1119, G3a #1120) into G4 left seven engine-free tests red in four files. 1. test_nz_spec_package.py::TestRulespecPin (two tests). G2's rulespec commit scanner read the closure's axiom-rules-engine commit (04315d94) as an unreviewed rulespec commit: input_surface held it under engine_commit in a mapping with no repository, and engine_optional_evidence put the hex in prose that cites the engine's src/rulespec.rs. The closure now records the engine as a foreign record, input_surface.engine = {repository: TheAxiomFoundation/axiom-rules-engine, commit}, the shape the committed surface fixture and axiom_rules_bindings.json already use. The evidence, and for consistency the undetermined-judgment reason (which the scanner did not flag), name "the axiom-rules-engine commit named in input_surface.engine" instead of repeating the hex. The scanner and the reviewed-commit set are unchanged. InputClosure gains surface_engine_repository; surface_engine_commit is kept, and from_dict refuses the old flat engine_commit shape. - New: a rulespec commit planted in the closure is still caught (beside the engine record, inside it under a rulespec key, in an entry reason, with the record relabelled as rulespec, and the engine hex back in the evidence prose). - New: the engine record's commit is pinned to the surface's engine commit by InputClosure.check, so the foreign record cannot carry a rulespec commit unnoticed. 2. test_nz_axiom_input_closure.py (three tests). G3a bound accommodation_supplement_cash_assets (fact:person.liquid_financial_assets, summed over the family), so its closure entry drops "awaiting" and is an ordinary concept-encoded input; the closure closes all 140 root inputs (60 + 28 + 13 + 39). The test that allowed only the cash-asset entry to await now requires that none does and that the entry is the mapping's summed liquid-asset binding. A new property checks that group encoding gives each family the sum of its members' liquid assets and that scenario S1 (MC10) zeroes it. 3. test_country_spec.py golden [nz] and test_spec_engine_country_bundles.py::test_nz_generation_zero_views_come_from_the_country_spec_seam. The merge listed axiom_input_closure.json in build/nz/country_package.json (the loader refuses undeclared files) without updating G2's pins. The nz golden is regenerated from the loaded spec after the last closure edit: the diff is only the new resource and its hash, country_package.json's hash and the fingerprint. The legacy_json resource set now includes the closure. Stale claims: the changelog fragment said the closure closes 139 of 140 inputs with the 140th awaiting G3a; it now says all 140. The axiom:nz cash-asset note said "no code applies it until a future unit-construction step builds Family units"; it now says encode_groups applies it on the units the unit-construction step builds, keeping the phrase G3a's note test pins. A closure-test comment said G2 "adds" benefit_unit_rule.json; G2 merged. Co-Authored-By: Claude Opus 5.5 --- .../unit-construction-group-encode.added.md | 2 +- .../build/nz/axiom_input_closure.json | 12 +-- .../shared/test_nz_axiom_input_closure.py | 88 +++++++++++++++---- .../shared/test_nz_spec_package.py | 48 ++++++++++ .../test_spec_engine_country_bundles.py | 1 + .../tests/golden/nz_country_spec.json | 6 +- .../adapters/axiom_concept_mappings/nz.json | 2 +- .../src/microcosm/frame/input_closure.py | 24 ++++- .../engine_free/shared/test_input_closure.py | 27 ++++++ 9 files changed, 181 insertions(+), 29 deletions(-) diff --git a/changelog.d/unit-construction-group-encode.added.md b/changelog.d/unit-construction-group-encode.added.md index 4cf6f2bf81..874933204d 100644 --- a/changelog.d/unit-construction-group-encode.added.md +++ b/changelog.d/unit-construction-group-encode.added.md @@ -1 +1 @@ -Add benefit-unit construction from concept pointers (`microcosm.frame.unit_construction`), group encoding that executes a mapping's group rules, state bindings and scenario knobs on built units (`ConceptMapping.encode_groups`), and Axiom input closures (`microcosm.frame.input_closure`). Extend the `axiom:nz` mapping with the Accommodation Supplement unit-composition flags, owner payments and relevant income, and the main-benefit rates income inputs, and commit the New Zealand input closure (`build/nz/axiom_input_closure.json`), which closes 139 of the 140 root inputs of the four bound modules against the committed input surface; the 140th, the Accommodation Supplement cash-asset input, awaits the liquid-asset binding (G3a). +Add benefit-unit construction from concept pointers (`microcosm.frame.unit_construction`), group encoding that executes a mapping's group rules, state bindings and scenario knobs on built units (`ConceptMapping.encode_groups`), and Axiom input closures (`microcosm.frame.input_closure`). Extend the `axiom:nz` mapping with the Accommodation Supplement unit-composition flags, owner payments and relevant income, and the main-benefit rates income inputs, and commit the New Zealand input closure (`build/nz/axiom_input_closure.json`), which closes all 140 root inputs of the four bound modules against the committed input surface, the Accommodation Supplement cash-asset input among them through the liquid-asset binding. diff --git a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json index d5b835c895..8116f1e802 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json +++ b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json @@ -12,7 +12,10 @@ "input_surface": { "fixture": "packages/microcosm-frame/tests/fixtures/axiom_input_surfaces/nz.json", "rulespec_commit": "6fe181fc4c65763150fe97501ef3443774e64869", - "engine_commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66" + "engine": { + "repository": "TheAxiomFoundation/axiom-rules-engine", + "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66" + } }, "rulespec_pin": { "commit": "8dc2507d5f10a0467c285fed54d018ee37246798", @@ -42,7 +45,7 @@ "A main benefit only where the rates module's income-tested net weekly payment for it is positive: entitlement treats the s 24 income test and the s 21 work gap as met (jobseeker_income_less_than_zero_rate_cutout true, jobseeker_in_full_time_employment false), and the Sole Parent Support and Supported Living Payment entitlement rules have no income term; the rates module tests the person's own income only." ] }, - "engine_optional_evidence": "axiom-rules-engine 04315d94 (the engine that generated the committed input surface) has no optional inputs. A RuleSpec module has no inputs section (RulesDocument holds format, imports, module, units, relations and rules; src/rulespec.rs:361-374), a rule-level default is rejected (RuleSpecError::UnsupportedDefault, src/rulespec.rs:91-94), and the dense batch call collects only the inputs the caller supplies (python-ext/src/lib.rs:458-466). A plain read of an absent input fails every row whose evaluation reaches it with EvalError::MissingInput (src/dense.rs:2790-2803), and that fails the request. The internal input_or_else node that would read a default (src/dense.rs:2805-2811) is never produced from RuleSpec text: formula lowering turns a name no rule defines into a plain input read (src/formula.rs:1400-1413). So no root input of these modules can be verified engine-optional, and this closure classes none as such: an input a run does not encode is defaulted here, with its reason.", + "engine_optional_evidence": "The axiom-rules-engine commit named in input_surface.engine (the engine that generated the committed input surface) has no optional inputs. A RuleSpec module has no inputs section (RulesDocument holds format, imports, module, units, relations and rules; src/rulespec.rs:361-374), a rule-level default is rejected (RuleSpecError::UnsupportedDefault, src/rulespec.rs:91-94), and the dense batch call collects only the inputs the caller supplies (python-ext/src/lib.rs:458-466). A plain read of an absent input fails every row whose evaluation reaches it with EvalError::MissingInput (src/dense.rs:2790-2803), and that fails the request. The internal input_or_else node that would read a default (src/dense.rs:2805-2811) is never produced from RuleSpec text: formula lowering turns a name no rule defines into a plain input read (src/formula.rs:1400-1413). So no root input of these modules can be verified engine-optional, and this closure classes none as such: an input a run does not encode is defaulted here, with its reason.", "undetermined_judgments": { "action": "refuse", "alternatives": [ @@ -50,7 +53,7 @@ "eligible" ], "new_default": true, - "reason": "Entitlement judgments come back as codes 1 (holds), -1 (does not hold) and 0 (undetermined). At axiom-rules-engine 04315d94 no dense leaf yields an undetermined value: dense input columns are booleans, integers, floats or text, an absent input is an error rather than an unknown, and and/or/not only pass an undetermined item along. With every root input encoded or defaulted by this closure, a 0 cannot arise, so takeup.assign@1 refuses any 0 it sees instead of guessing: a 0 would mean the engine or the closure changed. The alternatives a review could choose are treating 0 as not entitled (understates receipt) or as entitled (overstates it). microcosm.frame.input_closure.resolve_judgments implements all three." + "reason": "Entitlement judgments come back as codes 1 (holds), -1 (does not hold) and 0 (undetermined). At the axiom-rules-engine commit named in input_surface.engine, no dense leaf yields an undetermined value: dense input columns are booleans, integers, floats or text, an absent input is an error rather than an unknown, and and/or/not only pass an undetermined item along. With every root input encoded or defaulted by this closure, a 0 cannot arise, so takeup.assign@1 refuses any 0 it sees instead of guessing: a 0 would mean the engine or the closure changed. The alternatives a review could choose are treating 0 as not entitled (understates receipt) or as entitled (overstates it). microcosm.frame.input_closure.resolve_judgments implements all three." }, "knobs": [ { @@ -1085,8 +1088,7 @@ "entity": "Family", "input": "accommodation_supplement_cash_assets", "class": "encoded", - "encoded_by": "concept_binding", - "awaiting": "G3a (transport/liquid-assets-concept): the binding of fact:person.liquid_financial_assets, summed over the family" + "encoded_by": "concept_binding" }, { "rulespec_path": "nz/statutes/social_security/accommodation_supplement/core.yaml", diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py index e39660a906..89b45a70be 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py @@ -25,7 +25,7 @@ from microcosm.frame.adapters.axiom import _batch_from_table, axiom_concept_mapping from microcosm.frame.adapters.axiom_input_surface import load_axiom_input_surface -from microcosm.frame.concept_mapping import Allocation +from microcosm.frame.concept_mapping import Allocation, GroupRule, Identity from microcosm.frame.concepts import CONCEPTS, ContentBasis, split_for_transport from microcosm.frame.input_closure import ( ClosureClass, @@ -60,13 +60,13 @@ CLOSURE = InputClosure.from_dict(json.loads(CLOSURE_PATH.read_text(encoding="utf-8"))) SURFACE = load_axiom_input_surface(SURFACE_PATH) MAPPING = axiom_concept_mapping("nz") -#: Concept-encoded inputs whose binding lands in a later change (the cash-asset -#: binding arrives with the liquid-asset concept, work package G3a). The -#: closure records the change it waits for; when the binding lands, the -#: closure check reports the entry until its awaiting note is removed. +#: Concept-encoded inputs whose binding lands in a later change. None does now: +#: the cash-asset binding arrived with the liquid-asset concept (work package +#: G3a). An entry that awaits a binding the mapping already has is reported by +#: the closure check until its awaiting note is removed. AWAITING = {entry.ref for entry in CLOSURE.entries if entry.awaiting is not None} #: Stand-in unit rule for these tests. The NZ rule is spec data (method card -#: MC7, build/nz/benefit_unit_rule.json, which G2 adds); these properties +#: MC7, build/nz/benefit_unit_rule.json, which G2 added); these properties #: hold for any rule. RULE = BenefitUnitRule( entity="family", @@ -87,13 +87,23 @@ def test_the_closure_partitions_every_bound_module(self) -> None: assert CLOSURE.content_basis is ContentBasis.TRANSPORT assert CLOSURE.check(MAPPING, SURFACE) == () - def test_only_the_cash_asset_binding_awaits_a_later_change(self) -> None: - assert {ref.name for ref in AWAITING} <= { - "accommodation_supplement_cash_assets" - } - for entry in CLOSURE.entries: - if entry.awaiting is not None: - assert "G3a" in entry.awaiting + def test_the_cash_assets_are_the_bound_liquid_asset_input(self) -> None: + # G3a's binding has landed, so no input awaits a later change: the + # cash-asset input is concept-encoded like any other. + assert AWAITING == set() + (entry,) = ( + entry + for entry in CLOSURE.entries_for(AS) + if entry.input == "accommodation_supplement_cash_assets" + ) + assert entry.closure_class is ClosureClass.ENCODED + assert entry.encoded_by is EncodedBy.CONCEPT_BINDING + (binding,) = ( + binding for binding in MAPPING.bindings if binding.ref == entry.ref + ) + assert binding.concepts == ("fact:person.liquid_financial_assets",) + assert isinstance(binding.transform, Identity) + assert binding.group_rule is GroupRule.SUM_OVER_MEMBERS def test_the_receipt_layer_requirements_are_declared(self) -> None: requirements = _payload()["receipt_requirements"]["requirements"] @@ -130,14 +140,36 @@ def test_every_default_is_cited_and_listed_for_review(self) -> None: ) def test_no_input_is_claimed_engine_optional(self) -> None: - # axiom-rules-engine 04315d94 has no optional inputs; the closure says - # why and classes none as engine-optional. + # The axiom-rules-engine commit in input_surface.engine (04315d94) has + # no optional inputs; the closure says why and classes none as + # engine-optional. assert not any( entry.closure_class is ClosureClass.ENGINE_OPTIONAL for entry in CLOSURE.entries ) - assert "04315d94" in CLOSURE.engine_optional_evidence + assert "input_surface.engine" in CLOSURE.engine_optional_evidence assert CLOSURE.surface_engine_commit.startswith("04315d94") + assert CLOSURE.surface_engine_repository == ( + "TheAxiomFoundation/axiom-rules-engine" + ) + # The record is the committed surface's own engine record. + engine = json.loads(SURFACE_PATH.read_text(encoding="utf-8"))["engine"] + assert _payload()["input_surface"]["engine"] == { + "repository": engine["repository"], + "commit": engine["commit"], + } + + def test_the_engine_record_holds_the_surface_engine_commit(self) -> None: + # A record that names axiom-rules-engine is foreign to the rulespec + # commit scan (test_nz_spec_package.py), so the check pins its commit + # to the surface: any other commit there, a rulespec one included, is + # reported. + payload = _payload() + pin = payload["rulespec_pin"]["commit"] + payload["input_surface"]["engine"]["commit"] = pin + assert InputClosure.from_dict(payload).check(MAPPING, SURFACE) == ( + f"Surface engine {SURFACE.engine_commit} is not {pin}.", + ) def test_undetermined_judgments_are_refused(self) -> None: assert CLOSURE.undetermined.action is UndeterminedAction.REFUSE @@ -264,6 +296,30 @@ def test_encoding_and_defaults_supply_every_supplement_input( for name in expected - from_nodes - awaited: assert family[name].dtype.kind in "bif", name + @settings(max_examples=40, deadline=None) + @given( + tables=concept_frames(max_households=4, max_members=6, min_id=-(10**9)), + data=st.data(), + ) + def test_cash_assets_are_each_familys_summed_liquid_assets( + self, tables, data + ) -> None: + tables = _with_state(split_for_transport(tables)[0], data) + person = tables["person"] + _, membership = build_benefit_units(person, tables["household"], RULE) + summed = person["liquid_financial_assets"].groupby(membership).sum() + cash = "accommodation_supplement_cash_assets" + family = _family_inputs(tables).set_index("family_id") + assert np.allclose( + family[cash].to_numpy(), + summed.reindex(family.index).to_numpy(), + rtol=1e-12, + atol=1e-6, + ) + # Scenario S1 (method card MC10) switches the asset test off. + off = _family_inputs(tables, {"asset_test": False}) + assert (off[cash] == 0).all() + @settings(max_examples=40, deadline=None) @given(tables=concept_frames(max_households=4, max_members=6, min_id=-(10**9))) def test_encoding_and_defaults_supply_every_person_input(self, tables) -> None: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py index fe7320b6ca..2ca37223fb 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_spec_package.py @@ -811,6 +811,54 @@ def test_scanner_catches_the_phrasings_the_reviews_planted(self) -> None: "abad1dea", } + def test_a_rulespec_commit_planted_in_the_input_closure_is_caught(self) -> None: + # The closure names the axiom-rules-engine commit that generated the + # input surface in a foreign record (one naming that repository), so + # the scan passes it over; test_nz_axiom_input_closure.py pins that + # commit to the surface. A rulespec commit planted beside it is caught. + reviewed = reviewed_rulespec_commits() + clean = _package_payloads() + engine = clean["axiom_input_closure.json"]["input_surface"]["engine"] + assert engine["repository"] == "TheAxiomFoundation/axiom-rules-engine" + assert unreviewed_mentions(rulespec_commit_mentions(clean), reviewed) == set() + + def planted(edit: Callable[[dict[str, Any]], object]) -> set[str]: + payloads = copy.deepcopy(clean) + edit(payloads["axiom_input_closure.json"]) + return unreviewed_mentions(rulespec_commit_mentions(payloads), reviewed) + + assert planted( + lambda closure: closure["input_surface"].update(rulespec_commit="c0ffee1") + ) == {"c0ffee1"} + assert planted( + lambda closure: closure["input_surface"]["engine"].update( + rulespec_nz_commit="abad1dea" + ) + ) == {"abad1dea"} + assert planted( + lambda closure: closure["entries"][0].update( + reason="Set at rulespec-nz deadbee." + ) + ) == {"deadbee"} + # The record escapes the scan only while it names a non-rulespec + # repository. + assert planted( + lambda closure: closure["input_surface"]["engine"].update( + repository="TheAxiomFoundation/rulespec-nz" + ) + ) == {engine["commit"]} + # The engine evidence cites the engine's src/rulespec.rs, so the + # engine commit's hex in that prose reads as a rulespec mention. + short = engine["commit"][:8] + assert planted( + lambda closure: closure.update( + engine_optional_evidence=( + f"axiom-rules-engine {short}: " + + closure["engine_optional_evidence"] + ) + ) + ) == {short} + @settings(max_examples=300, deadline=None) @given( commit=_HEX40_TEXT, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py index 348776d52c..f39358e700 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_country_bundles.py @@ -222,6 +222,7 @@ def test_nz_generation_zero_views_come_from_the_country_spec_seam() -> None: assert {row.path for row in spec.resource_rows if row.kind == "legacy_json"} == { "as_area_crosswalk.json", "as_rate_bridge.json", + "axiom_input_closure.json", "axiom_rules_bindings.json", "benefit_unit_rule.json", "currency_bridge.json", diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json index 2f0ff95ad4..d7cfeb2795 100644 --- a/packages/microcosm-build/tests/golden/nz_country_spec.json +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -1,6 +1,6 @@ { "country": "nz", - "fingerprint": "df8850d5893e9311882573ff6b4baa41c9dda41f6838d9a140199ddf0cc7fd48", + "fingerprint": "cdee3f8a9363d70be182a0338854200353b5b99a28d4bf56cc28cb3bf97c7c18", "gate_ids": [ "calibration_per_family_fit", "national_and_region_admin_aggregates", @@ -36,9 +36,10 @@ "resource_hashes": { "as_area_crosswalk.json": "cfb76c99898204598b0609e5462b2fba9d081872a13ac5dd973cb406bb6f7469", "as_rate_bridge.json": "5eea897159508fb71820f22cd8ccf4993075a750462bd97312d9e89ab01cba15", + "axiom_input_closure.json": "bc9e024400c4124028f62313785a70752df2977078857b5502939a01339aef3a", "axiom_rules_bindings.json": "72905af77132ff6a3c01ada4b6210be4c369d9c6dc49dc8c1b673e4bac0b5fb1", "benefit_unit_rule.json": "9e507e78f180566a1eece74da2a35d88478dbe13487c3a64ee2b76c9ecc7a653", - "country_package.json": "b83de9e12238b41481810ddb7fbdbaaa71d457cbfa9da83f3f53cbf479cee43b", + "country_package.json": "6c4c1d19a0666958a4dc1079e5363084a347e817fc0c9cca59eada45bdc18873", "currency_bridge.json": "aa6a4b286a6a1a015088f32ac013148553422c3d45eeef7d7a15af08388857ca", "export_contract.json": "f0629a3990a47f3f94114c4d7b6680636ef7b1bd34bb8e950976d3065f029fda", "gates.json": "3d709d88bd40198b61b9a1745b6f4481625145aa13a7414d700726c50159a0ae", @@ -65,6 +66,7 @@ "spec/vintages.yaml", "as_area_crosswalk.json", "as_rate_bridge.json", + "axiom_input_closure.json", "axiom_rules_bindings.json", "benefit_unit_rule.json", "currency_bridge.json", diff --git a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json index 41081a8a7a..c7efec54fb 100644 --- a/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json +++ b/packages/microcosm-frame/src/microcosm/frame/adapters/axiom_concept_mappings/nz.json @@ -512,7 +512,7 @@ "engine_input": "accommodation_supplement_cash_assets", "group_rule": "sum_over_members", "module": "nz/statutes/social_security/accommodation_supplement/core.yaml", - "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_cash_assets_within_limit (source: Social Security Act 2018 s 65(1)(b); Social Security Regulations 2018 reg 15). It holds when accommodation_supplement_cash_assets is at most accommodation_supplement_cash_asset_limit_partnered_or_with_children (16200, reg 15(1)(a)) for a family in a relationship or with dependent children, and at most accommodation_supplement_cash_asset_limit_other (8100, reg 15(1)(b)) otherwise, and accommodation_supplement_non_rate_entitlement_requirements (ss 65 to 67) requires it. The module does not define cash assets. The Act does, in s 68(2), which Schedule 2 applies for the purposes of the accommodation supplement: money saved, invested or banked with a bank or other institution; money invested in securities, bonds or debentures, or advanced on mortgage; money withdrawn from a KiwiSaver scheme; and money invested in shares in a partnership, company or other body. It excludes KiwiSaver contributions and member's interests, and anything the regulations exempt (reg 16 and Schedule 8 Part 1). Under s 69(2), which reg 15(3) restates, an applicant's assets include those of their spouse or partner, so the binding sums members' amounts onto the Family entity. It is deferred: no code applies it until a future unit-construction step builds Family units, and that step's unit rule will decide who is a member. Differences: the concept leaves out money advanced on mortgage, withdrawn KiwiSaver money held as cash, shares in a partnership or company the person runs, which s 68(2)(a)(iv) counts as money invested in shares in a partnership or limited liability company or other incorporated or unincorporated body (a sole trader's non-cash business equity is in neither), and a member's interest in a non-KiwiSaver superannuation scheme, which Schedule 8 item 10 exempts only while a lock-in clause prevents withdrawal (Schedule 8 cl 25); the other Schedule 8 exemptions are not applied; if the unit rule makes a dependent child a Family member, the child's assets will be summed too, although s 69(2) adds only a spouse's or partner's, and if it leaves out a spouse or partner in long-term residential care, whose assets s 69(2A) still counts, their assets will be missed; and the concept is the stock at the survey reference date, while the test applies when the supplement is assessed.", + "note": "Consumed in nz/statutes/social_security/accommodation_supplement/core.yaml by accommodation_supplement_cash_assets_within_limit (source: Social Security Act 2018 s 65(1)(b); Social Security Regulations 2018 reg 15). It holds when accommodation_supplement_cash_assets is at most accommodation_supplement_cash_asset_limit_partnered_or_with_children (16200, reg 15(1)(a)) for a family in a relationship or with dependent children, and at most accommodation_supplement_cash_asset_limit_other (8100, reg 15(1)(b)) otherwise, and accommodation_supplement_non_rate_entitlement_requirements (ss 65 to 67) requires it. The module does not define cash assets. The Act does, in s 68(2), which Schedule 2 applies for the purposes of the accommodation supplement: money saved, invested or banked with a bank or other institution; money invested in securities, bonds or debentures, or advanced on mortgage; money withdrawn from a KiwiSaver scheme; and money invested in shares in a partnership, company or other body. It excludes KiwiSaver contributions and member's interests, and anything the regulations exempt (reg 16 and Schedule 8 Part 1). Under s 69(2), which reg 15(3) restates, an applicant's assets include those of their spouse or partner, so the binding sums members' amounts onto the Family entity. Group encoding (ConceptMapping.encode_groups) applies it to the Family units the unit-construction step (microcosm.frame.unit_construction) builds under a declared benefit-unit rule, which decides who is a member; no New Zealand build applies it yet, and the country pack's rule (method card MC7) awaits the D1 method review. Differences: the concept leaves out money advanced on mortgage, withdrawn KiwiSaver money held as cash, shares in a partnership or company the person runs, which s 68(2)(a)(iv) counts as money invested in shares in a partnership or limited liability company or other incorporated or unincorporated body (a sole trader's non-cash business equity is in neither), and a member's interest in a non-KiwiSaver superannuation scheme, which Schedule 8 item 10 exempts only while a lock-in clause prevents withdrawal (Schedule 8 cl 25); the other Schedule 8 exemptions are not applied; if the unit rule makes a dependent child a Family member, the child's assets will be summed too, although s 69(2) adds only a spouse's or partner's, and if it leaves out a spouse or partner in long-term residential care, whose assets s 69(2A) still counts, their assets will be missed; and the concept is the stock at the survey reference date, while the test applies when the supplement is assessed.", "relation": "approximate", "transform": { "kind": "identity" diff --git a/packages/microcosm-frame/src/microcosm/frame/input_closure.py b/packages/microcosm-frame/src/microcosm/frame/input_closure.py index 7e7560dfb6..6d9658658a 100644 --- a/packages/microcosm-frame/src/microcosm/frame/input_closure.py +++ b/packages/microcosm-frame/src/microcosm/frame/input_closure.py @@ -439,6 +439,11 @@ class InputClosure: mapping_engine_version: That mapping's ``engine_version``. surface_rulespec_commit: The rulespec commit the input surface was generated from. + surface_engine_repository: The repository of the engine that + generated it. The engine commit is serialized in a record that + names this repository, which marks it as the engine's commit, not + a rulespec commit; :meth:`check` compares the commit with the + surface, not the repository. surface_engine_commit: The engine commit that generated it. modules: Module path -> SHA-256 of the module bytes reviewed. engine_optional_evidence: What the engine does with an absent input, @@ -454,6 +459,7 @@ class InputClosure: mapping_engine: str mapping_engine_version: str surface_rulespec_commit: str + surface_engine_repository: str surface_engine_commit: str modules: Mapping[str, str] engine_optional_evidence: str @@ -706,7 +712,10 @@ def to_dict(self) -> dict[str, object]: }, "input_surface": { "rulespec_commit": self.surface_rulespec_commit, - "engine_commit": self.surface_engine_commit, + "engine": { + "repository": self.surface_engine_repository, + "commit": self.surface_engine_commit, + }, }, "rulespec_paths": dict(self.modules), "engine_optional_evidence": self.engine_optional_evidence, @@ -756,12 +765,18 @@ def from_dict(cls, data: Mapping[str, object]) -> InputClosure: surface = _record_fields( data["input_surface"], "closure input surface", - required=("rulespec_commit", "engine_commit"), + required=("rulespec_commit", "engine"), optional=("fixture",), ) + engine = _record_fields( + surface["engine"], + "closure input-surface engine", + required=("repository", "commit"), + ) _text_fields(mapping, "closure mapping", ("engine", "engine_version")) + _text_fields(surface, "closure input surface", ("rulespec_commit",)) _text_fields( - surface, "closure input surface", ("rulespec_commit", "engine_commit") + engine, "closure input-surface engine", ("repository", "commit") ) modules = data["rulespec_paths"] if not isinstance(modules, Mapping) or not all( @@ -794,7 +809,8 @@ def from_dict(cls, data: Mapping[str, object]) -> InputClosure: mapping_engine=mapping["engine"], mapping_engine_version=mapping["engine_version"], surface_rulespec_commit=surface["rulespec_commit"], - surface_engine_commit=surface["engine_commit"], + surface_engine_repository=engine["repository"], + surface_engine_commit=engine["commit"], modules=modules, engine_optional_evidence=evidence, undetermined=UndeterminedPolicy(**policy), diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py b/packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py index ab849fea22..7f0bd82e6f 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_input_closure.py @@ -160,6 +160,7 @@ def _closure(**change) -> InputClosure: "mapping_engine": "axiom:zz", "mapping_engine_version": "r1", "surface_rulespec_commit": "r1", + "surface_engine_repository": "TheAxiomFoundation/axiom-rules-engine", "surface_engine_commit": "e1", "modules": {MODULE: SHA}, "engine_optional_evidence": "the engine has no optional inputs", @@ -516,10 +517,36 @@ def test_review_fields_beside_the_closure_are_allowed(self) -> None: payload["input_surface"]["fixture"] = "path/to/surface.json" assert InputClosure.from_dict(payload).to_dict() == _closure().to_dict() + def test_the_surface_engine_is_a_record_naming_its_repository(self) -> None: + # The engine commit is not a rulespec commit, so it sits in a record + # that names the engine's repository. + payload = _closure().to_dict() + assert payload["input_surface"] == { + "rulespec_commit": "r1", + "engine": { + "repository": "TheAxiomFoundation/axiom-rules-engine", + "commit": "e1", + }, + } + again = InputClosure.from_dict(payload) + assert again.surface_engine_repository == ( + "TheAxiomFoundation/axiom-rules-engine" + ) + assert again.surface_engine_commit == "e1" + @pytest.mark.parametrize( "change", [ {"format": "x"}, + {"input_surface": {"rulespec_commit": "r1", "engine_commit": "e1"}}, + {"input_surface": {"rulespec_commit": "r1", "engine": "e1"}}, + {"input_surface": {"rulespec_commit": "r1", "engine": {"commit": "e1"}}}, + { + "input_surface": { + "rulespec_commit": "r1", + "engine": {"repository": 7, "commit": "e1"}, + } + }, {"rulespec_paths": {"m": 1}}, {"knobs": {}}, {"entries": [{"module": "m"}]}, From 8cdc8848a97c73fea54510d896472bc1e8e532dd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 09:56:01 -0400 Subject: [PATCH 4/5] Pin the engine source files the NZ closure cites by line The closure cites axiom-rules-engine source lines (src/rulespec.rs, src/dense.rs, src/formula.rs, python-ext/src/lib.rs) in engine_optional_evidence. A new top-level engine_source_pins records each cited file with the repository and commit of input_surface.engine, so every line citation names one file version. This is what the cross-country citation guard in microcosm#1146 accepts (a mapping with path and commit for the file). Putting the sha inline in the evidence string instead would trip G2's rulespec-commit scanner, which reads hex runs in any string that mentions RuleSpec; the pins are foreign records (repository is axiom-rules-engine), which that scanner skips by design. InputClosure.from_dict allows unknown top-level fields, so the loader is unchanged. A new test requires every cited .rs file to be pinned at the surface engine's repository and commit, and every pin to be cited; it fails without the pins. The NZ country-spec golden is regenerated (closure hash and fingerprint only). Tests: test_nz_axiom_input_closure.py 17 passed 1 skipped, test_nz_spec_package.py 71, test_spec_only_country_packages.py 8, test_country_spec.py 135. Co-Authored-By: Claude Opus 5.5 --- .../build/nz/axiom_input_closure.json | 25 +++++++++++++++++++ .../shared/test_nz_axiom_input_closure.py | 18 +++++++++++++ .../tests/golden/nz_country_spec.json | 4 +-- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json index 8116f1e802..25e890d02f 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json +++ b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json @@ -46,6 +46,31 @@ ] }, "engine_optional_evidence": "The axiom-rules-engine commit named in input_surface.engine (the engine that generated the committed input surface) has no optional inputs. A RuleSpec module has no inputs section (RulesDocument holds format, imports, module, units, relations and rules; src/rulespec.rs:361-374), a rule-level default is rejected (RuleSpecError::UnsupportedDefault, src/rulespec.rs:91-94), and the dense batch call collects only the inputs the caller supplies (python-ext/src/lib.rs:458-466). A plain read of an absent input fails every row whose evaluation reaches it with EvalError::MissingInput (src/dense.rs:2790-2803), and that fails the request. The internal input_or_else node that would read a default (src/dense.rs:2805-2811) is never produced from RuleSpec text: formula lowering turns a name no rule defines into a plain input read (src/formula.rs:1400-1413). So no root input of these modules can be verified engine-optional, and this closure classes none as such: an input a run does not encode is defaulted here, with its reason.", + "engine_source_pins": { + "note": "The axiom-rules-engine source files that engine_optional_evidence and undetermined_judgments cite by line, pinned at input_surface.engine.commit so each line citation names one exact file version.", + "pins": [ + { + "repository": "TheAxiomFoundation/axiom-rules-engine", + "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "path": "python-ext/src/lib.rs" + }, + { + "repository": "TheAxiomFoundation/axiom-rules-engine", + "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "path": "src/dense.rs" + }, + { + "repository": "TheAxiomFoundation/axiom-rules-engine", + "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "path": "src/formula.rs" + }, + { + "repository": "TheAxiomFoundation/axiom-rules-engine", + "commit": "04315d94d04efd61e27665ab3aa300d64fdf3a66", + "path": "src/rulespec.rs" + } + ] + }, "undetermined_judgments": { "action": "refuse", "alternatives": [ diff --git a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py index 89b45a70be..cbe227def6 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_nz_axiom_input_closure.py @@ -15,6 +15,7 @@ import importlib.util import json import os +import re from pathlib import Path import numpy as np @@ -83,6 +84,23 @@ def _payload() -> dict: class TestClosure: + def test_every_engine_line_citation_is_pinned_at_the_surface_engine(self) -> None: + # A line citation into axiom-rules-engine names one file version only + # when a pin records that file at input_surface.engine's commit. + payload = _payload() + engine = payload["input_surface"]["engine"] + pins = payload["engine_source_pins"]["pins"] + pinned = {pin["path"] for pin in pins} + assert all( + pin["repository"] == engine["repository"] + and pin["commit"] == engine["commit"] + for pin in pins + ) + cited = set(re.findall(r"((?:[\w-]+/)*[\w-]+\.rs):\d", json.dumps(payload))) + assert cited, "the closure cites engine source lines" + assert cited <= pinned + assert pinned <= cited, "every pin is cited" + def test_the_closure_partitions_every_bound_module(self) -> None: assert CLOSURE.content_basis is ContentBasis.TRANSPORT assert CLOSURE.check(MAPPING, SURFACE) == () diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json index d7cfeb2795..04b6f8049e 100644 --- a/packages/microcosm-build/tests/golden/nz_country_spec.json +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -1,6 +1,6 @@ { "country": "nz", - "fingerprint": "cdee3f8a9363d70be182a0338854200353b5b99a28d4bf56cc28cb3bf97c7c18", + "fingerprint": "0784fa546d409d9285a9c28002cb4ce021553f0eaca6acad6779752797615e38", "gate_ids": [ "calibration_per_family_fit", "national_and_region_admin_aggregates", @@ -36,7 +36,7 @@ "resource_hashes": { "as_area_crosswalk.json": "cfb76c99898204598b0609e5462b2fba9d081872a13ac5dd973cb406bb6f7469", "as_rate_bridge.json": "5eea897159508fb71820f22cd8ccf4993075a750462bd97312d9e89ab01cba15", - "axiom_input_closure.json": "bc9e024400c4124028f62313785a70752df2977078857b5502939a01339aef3a", + "axiom_input_closure.json": "9a07484c1751b855290491b396d9ff5d73d8bfaf03b084b76042119d06312f3a", "axiom_rules_bindings.json": "72905af77132ff6a3c01ada4b6210be4c369d9c6dc49dc8c1b673e4bac0b5fb1", "benefit_unit_rule.json": "9e507e78f180566a1eece74da2a35d88478dbe13487c3a64ee2b76c9ecc7a653", "country_package.json": "6c4c1d19a0666958a4dc1079e5363084a347e817fc0c9cca59eada45bdc18873", From a545fd72c1c29aa9136b80f0dce92901e45b041c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 8 Oct 2026 19:20:56 -0400 Subject: [PATCH 5/5] Refuse unsigned ids int64 cannot hold; read state bindings once Benefit-unit construction converted every id and pointer to int64 with a bare cast. The concept-frame contract accepts unsigned ids, so a household id of 2**63 became -2**63: its unit nested in no household and the household counted no unit. The validator matches pointers as int64 too, so a partner pointer of 2**64 - 1 passed as person -1. build_benefit_units now refuses, before validation, any id or pointer column holding an unsigned id above 2**63 - 1, naming the columns. Every other id conversion in the module (roles, attributes, membership) goes through one exact helper that refuses such ids and non-integer ids. Ids that fit are kept exactly, and the unit outputs stay int64. encode_groups compares membership ids as int64 too. A uint64 membership naming unit 2**64 - 1 was read as unit -1 and accepted, and so was a person renamed 2**64 - 1 under a head column of -1. It now refuses unsigned ids above 2**63 - 1 in the membership columns and the frame's person id columns. The bound check is shared with unit construction. encode_groups iterated state_bindings twice: once to encode and again to find the columns a knob may replace. A one-shot iterator was used up by the first pass, so a valid area knob was refused ("Knobs replace state columns no binding reads"). The bindings are now materialized once on entry. Tests: boundary regressions at 2**63 - 1 (kept exactly: nesting, identity and unit counts), at 2**63 and at 2**64 - 1 (refused), plus the wrapped partner pointer, the wrapped membership and the renamed person. Hypothesis properties: ids of every accepted integer dtype, signed and unsigned mixed, are kept exactly or refused, and build the same units, roles and attributes as the same ids typed int64. Membership ids retyped as uint64, UInt64 or Int64 encode exactly as the int64 ones, or are refused. State bindings given as a list, iterator or generator encode, or are refused, exactly as their tuple. Follow-ups from the independent verification of this fix (APPROVE_WITH_NITS): - the membership check also refuses a partner pointer int64 cannot hold (person.partner_person_id of 2**64 - 1 was read as person -1 and passed as that head's partner); a regression covers it and the int64 dangling-pointer control; - the closure's notes say takeup.assign@1 is declared but not yet implemented (three sentences read as if it ran); the NZ country-spec golden is regenerated for that text. Still open, outside this PR: concepts._check_pointers.positions and _Context.person_rows on main cast uint64 pointers to int64 the same way. Final targeted runs: test_unit_construction.py 38, test_group_encode.py 78, test_input_closure.py 52, test_concept_mapping.py 168, test_nz_axiom_input_closure.py 17 passed 1 skipped, test_nz_spec_package.py 71, test_country_spec.py 135, test_spec_engine_country_bundles.py 21, test_spec_only_country_packages.py 8. Co-Authored-By: Claude Opus 5.5 --- .../build/nz/axiom_input_closure.json | 6 +- .../tests/golden/nz_country_spec.json | 4 +- .../src/microcosm/frame/concept_mapping.py | 56 ++++- .../src/microcosm/frame/unit_construction.py | 94 ++++++-- .../engine_free/shared/test_group_encode.py | 170 ++++++++++++- .../shared/test_unit_construction.py | 225 +++++++++++++++++- 6 files changed, 522 insertions(+), 33 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json index 25e890d02f..61a9f440ee 100644 --- a/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json +++ b/packages/microcosm-build/src/microcosm/build/nz/axiom_input_closure.json @@ -35,10 +35,10 @@ "person.receives_supported_living_payment", "person.receives_nz_super" ], - "note": "The person-level receipt flags the receipt layer (takeup.assign@1) writes and the state bindings read. Their names are a contract with that kernel." + "note": "The person-level receipt flags the receipt layer (takeup.assign@1, declared but not yet implemented) is to write and the state bindings read. Their names are a contract with that kernel." }, "receipt_requirements": { - "note": "What the receipt layer (takeup.assign@1) must enforce because the closure's entitlement defaults leave it to receipt. Each is a modelling default for D1.", + "note": "What the receipt layer (takeup.assign@1, declared but not yet implemented) must enforce because the closure's entitlement defaults leave it to receipt. Each is a modelling default for D1.", "requirements": [ "At most one main benefit (Jobseeker Support, Sole Parent Support, Supported Living Payment) per person: Social Security Act 2018 s 18; main_benefit_receiving_another_main_benefit is defaulted false.", "Never New Zealand Superannuation and a main benefit for the same person: s 18; main_benefit_receiving_new_zealand_superannuation is defaulted false on transported frames.", @@ -78,7 +78,7 @@ "eligible" ], "new_default": true, - "reason": "Entitlement judgments come back as codes 1 (holds), -1 (does not hold) and 0 (undetermined). At the axiom-rules-engine commit named in input_surface.engine, no dense leaf yields an undetermined value: dense input columns are booleans, integers, floats or text, an absent input is an error rather than an unknown, and and/or/not only pass an undetermined item along. With every root input encoded or defaulted by this closure, a 0 cannot arise, so takeup.assign@1 refuses any 0 it sees instead of guessing: a 0 would mean the engine or the closure changed. The alternatives a review could choose are treating 0 as not entitled (understates receipt) or as entitled (overstates it). microcosm.frame.input_closure.resolve_judgments implements all three." + "reason": "Entitlement judgments come back as codes 1 (holds), -1 (does not hold) and 0 (undetermined). At the axiom-rules-engine commit named in input_surface.engine, no dense leaf yields an undetermined value: dense input columns are booleans, integers, floats or text, an absent input is an error rather than an unknown, and and/or/not only pass an undetermined item along. With every root input encoded or defaulted by this closure, a 0 cannot arise, so the receipt kernel takeup.assign@1, once implemented, is to refuse any 0 it sees instead of guessing: a 0 would mean the engine or the closure changed. The alternatives a review could choose are treating 0 as not entitled (understates receipt) or as entitled (overstates it). microcosm.frame.input_closure.resolve_judgments implements all three." }, "knobs": [ { diff --git a/packages/microcosm-build/tests/golden/nz_country_spec.json b/packages/microcosm-build/tests/golden/nz_country_spec.json index 04b6f8049e..c8e85b0fda 100644 --- a/packages/microcosm-build/tests/golden/nz_country_spec.json +++ b/packages/microcosm-build/tests/golden/nz_country_spec.json @@ -1,6 +1,6 @@ { "country": "nz", - "fingerprint": "0784fa546d409d9285a9c28002cb4ce021553f0eaca6acad6779752797615e38", + "fingerprint": "99d7cf58cc7ec8bd118559baf82b70fe39f91b64b27671e1dac22e78fdd92522", "gate_ids": [ "calibration_per_family_fit", "national_and_region_admin_aggregates", @@ -36,7 +36,7 @@ "resource_hashes": { "as_area_crosswalk.json": "cfb76c99898204598b0609e5462b2fba9d081872a13ac5dd973cb406bb6f7469", "as_rate_bridge.json": "5eea897159508fb71820f22cd8ccf4993075a750462bd97312d9e89ab01cba15", - "axiom_input_closure.json": "9a07484c1751b855290491b396d9ff5d73d8bfaf03b084b76042119d06312f3a", + "axiom_input_closure.json": "c3c164f437b05d598af7e604d2cb46365a8323a8043da49b08c5e0cdf32e2ec1", "axiom_rules_bindings.json": "72905af77132ff6a3c01ada4b6210be4c369d9c6dc49dc8c1b673e4bac0b5fb1", "benefit_unit_rule.json": "9e507e78f180566a1eece74da2a35d88478dbe13487c3a64ee2b76c9ecc7a653", "country_package.json": "6c4c1d19a0666958a4dc1079e5363084a347e817fc0c9cca59eada45bdc18873", diff --git a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py index 133340d334..86908ead27 100644 --- a/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py +++ b/packages/microcosm-frame/src/microcosm/frame/concept_mapping.py @@ -1053,8 +1053,9 @@ class GroupMembership: (:func:`microcosm.frame.unit_construction.benefit_unit_membership`). :meth:`ConceptMapping.encode_groups` checks them against the concept frame before using them: membership aligned to the person index, integer - ids, members in the household their unit declares, one head per unit and - at most one partner, who is the head's partner by the pointers. + ids that int64 can hold, members in the household their unit declares, + one head per unit and at most one partner, who is the head's partner by + the pointers. Attributes: entity: The frame group entity the units are (``family``). The unit @@ -1792,21 +1793,26 @@ def encode_groups( memberships: Engine group entity (``Family``) -> its units. modules: The RuleSpec modules to encode for; ``None`` encodes all. knobs: Declared encoding alternatives; ``None`` is the default. - state_bindings: Group inputs fed from model state. + state_bindings: Group inputs fed from model state, as any + iterable; a one-shot iterator is read once. shares: Share and fraction parameters, as for :meth:`encode`. take_up_rates: Take-up rates, as for :meth:`encode`. Raises: ValueError: If a membership does not match the frame (a person outside every unit, a unit without exactly one head, members - in two households), a state binding feeds an input a concept - binding or another state binding also feeds, a knob names an - input this call does not produce or cannot change, or a value - a transform needs is missing. + in two households, an unsigned id above ``2**63 - 1``, which + its int64 comparison cannot hold), a state binding feeds an + input a concept binding or another state binding also feeds, + a knob names an input this call does not produce or cannot + change, or a value a transform needs is missing. """ knobs = knobs or GroupKnobs() selected = None if modules is None else frozenset(modules) + # Read twice (to encode, then to find the columns knobs may replace), + # so a one-shot iterator is materialized once, here. + state_bindings = tuple(state_bindings) shares = dict(shares or {}) rates = dict(take_up_rates or {}) person = tables["person"] @@ -2175,6 +2181,19 @@ def _state_column(column: str) -> tuple[str, str]: return entity, name +#: The largest id a unit table carries: units hold and compare ids as int64. +_INT64_MAX = int(np.iinfo(np.int64).max) + + +def _exceeds_int64(values: pd.Series) -> bool: + """Whether an unsigned integer column holds an id int64 cannot hold.""" + + if values.dtype.kind != "u": + return False + present = values.dropna() + return len(present) > 0 and int(present.max()) > _INT64_MAX + + class _Units: """A membership checked against a concept frame, as row positions.""" @@ -2210,6 +2229,29 @@ def __init__(self, membership: GroupMembership, context: _Context) -> None: raise ValueError( f"{membership.entity} {name} must be integers, not {values.dtype}." ) + # The ids are compared as int64 below. An unsigned id int64 cannot + # hold would wrap onto another id (2**64 - 1 onto -1), so a + # membership naming no unit could pass; it is refused instead. + wide = [ + name + for name, values in ( + ("person unit ids", membership.person_unit), + *((column, units[column]) for column in columns), + (f"person.{_PERSON_ID}", person[_PERSON_ID]), + (f"person.{_PERSON_HOUSEHOLD_ID}", person[_PERSON_HOUSEHOLD_ID]), + *( + (f"person.{column}", person[column]) + for column in ("partner_person_id",) + if column in person.columns + ), + ) + if _exceeds_int64(values) + ] + if wide: + raise ValueError( + f"{membership.entity} ids are compared as int64, and {wide} hold " + f"unsigned ids above {_INT64_MAX}, which int64 cannot represent." + ) ids = units[membership.id_column].to_numpy(dtype=np.int64) if len(set(ids.tolist())) != len(ids): raise ValueError(f"{membership.entity} ids must be unique.") diff --git a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py index 535f96bda4..9cd4ecad4c 100644 --- a/packages/microcosm-frame/src/microcosm/frame/unit_construction.py +++ b/packages/microcosm-frame/src/microcosm/frame/unit_construction.py @@ -56,6 +56,12 @@ those columns rather than rely on the id's value. Units carry no weight of their own: a unit takes its household's weight through its members (:meth:`~microcosm.frame.bundle.Frame.resolve_weights`). + +The unit table and the membership hold ids as ``int64``. The concept-frame +contract accepts ids of any integer dtype, unsigned ones included; every id +and pointer this module reads is carried over exactly, and an unsigned id +above ``2**63 - 1``, which ``int64`` cannot hold, is refused before any +conversion rather than wrapped to a negative id. """ from __future__ import annotations @@ -69,7 +75,12 @@ import numpy as np import pandas as pd -from microcosm.frame.concept_mapping import GroupMembership, UnitRole +from microcosm.frame.concept_mapping import ( + _INT64_MAX, + GroupMembership, + UnitRole, + _exceeds_int64, +) from microcosm.frame.concepts import ( HOUSEHOLD_ID_COLUMN, PERSON_HOUSEHOLD_ID_COLUMN, @@ -101,6 +112,11 @@ _HOUSEHOLD_ID = HOUSEHOLD_ID_COLUMN _ENTITY_NAME = re.compile(r"^[a-z][a-z0-9_]*$") _POINTERS = ("partner_person_id", "parent_1_person_id", "parent_2_person_id") +#: Every id and pointer column unit construction reads, by concept table. +_ID_COLUMNS = { + "person": (_PERSON_ID, _PERSON_HOUSEHOLD_ID, *_POINTERS), + "household": (_HOUSEHOLD_ID, "reference_person_id"), +} class UnparentedChildPlacement(StrEnum): @@ -359,8 +375,9 @@ def build_benefit_units( ValueError: If a needed column is missing, the frame breaks a concept-frame contract (pointers that dangle or leave the household, asymmetric partners, parent cycles, a reference person - outside their household, bad ages or hours), or the rule refuses - a child's placement. + outside their household, bad ages or hours), an id or pointer + is an unsigned integer above ``2**63 - 1`` (the unit tables hold + ``int64`` ids), or the rule refuses a child's placement. """ rows = _Pointers.read(person, household, rule) @@ -400,15 +417,17 @@ def benefit_unit_roles( :class:`~microcosm.frame.concept_mapping.UnitRole`. """ - ids = person[_PERSON_ID].to_numpy(dtype=np.int64) + ids = _int64_ids(person[_PERSON_ID], f"person.{_PERSON_ID}") units = _unit_rows(family, membership, rule) - head_of = family[rule.head_column].to_numpy(dtype=np.int64)[units] + head_of = _int64_ids(family[rule.head_column], rule.head_column)[units] is_head = ids == head_of # Partner presence is a mask, not a sentinel id: any integer, -1 # included, can be a person id. present = person["partner_person_id"].notna().to_numpy() partner = np.zeros(len(ids), dtype=np.int64) - partner[present] = person["partner_person_id"][present].to_numpy(dtype=np.int64) + partner[present] = _int64_ids( + person["partner_person_id"][present], "person.partner_person_id" + ) head_row = pd.Index(ids).get_indexer(head_of) found = head_row >= 0 head_row = np.where(found, head_row, 0) @@ -488,7 +507,7 @@ def benefit_unit_attributes( ) return pd.DataFrame( { - rule.id_column: family[rule.id_column].to_numpy(dtype=np.int64), + rule.id_column: _int64_ids(family[rule.id_column], rule.id_column), "n_members": n_members, "n_adults": n_adults, "n_dependent_children": n_children, @@ -554,6 +573,19 @@ def read( "person": person.loc[:, person_columns], "household": household.loc[:, [_HOUSEHOLD_ID, "reference_person_id"]], } + # Validation matches pointers as int64 too, so an id int64 cannot + # hold is refused before it, not only before this module's casts. + wide = [ + f"{entity}.{column}" + for entity, columns in _ID_COLUMNS.items() + for column in columns + if _exceeds_int64(subset[entity][column]) + ] + if wide: + raise ValueError( + f"Benefit units hold ids as int64, and {wide} hold unsigned ids " + f"above {_INT64_MAX}, which int64 cannot represent." + ) violations = validate_concept_tables(subset) if violations: detail = "; ".join( @@ -562,32 +594,36 @@ def read( for item in violations ) raise ValueError(f"Benefit units refuse an invalid concept frame: {detail}") - ids = person[_PERSON_ID].to_numpy(dtype=np.int64) + ids = _int64_ids(person[_PERSON_ID], f"person.{_PERSON_ID}") index = pd.Index(ids) - def rows(values: pd.Series) -> np.ndarray: + def rows(table: pd.DataFrame, entity: str, column: str) -> np.ndarray: + values = table[column] present = values.notna().to_numpy() found = np.full(len(values), -1, dtype=np.int64) if present.any(): found[present] = index.get_indexer( - values[present].to_numpy(dtype=np.int64) + _int64_ids(values[present], f"{entity}.{column}") ) return found + household_ids = _int64_ids( + person[_PERSON_HOUSEHOLD_ID], f"person.{_PERSON_HOUSEHOLD_ID}" + ) household_rows = pd.Index( - household[_HOUSEHOLD_ID].to_numpy(dtype=np.int64) - ).get_indexer(person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64)) - reference_by_household = rows(household["reference_person_id"]) + _int64_ids(household[_HOUSEHOLD_ID], f"household.{_HOUSEHOLD_ID}") + ).get_indexer(household_ids) + reference_by_household = rows(household, "household", "reference_person_id") return cls( ids=ids, - household_ids=person[_PERSON_HOUSEHOLD_ID].to_numpy(dtype=np.int64), + household_ids=household_ids, age=person["age"].to_numpy(dtype=np.float64), hours=None if test is None else person["usual_weekly_hours"].to_numpy(dtype=np.float64), - partner=rows(person["partner_person_id"]), - parent_1=rows(person["parent_1_person_id"]), - parent_2=rows(person["parent_2_person_id"]), + partner=rows(person, "person", "partner_person_id"), + parent_1=rows(person, "person", "parent_1_person_id"), + parent_2=rows(person, "person", "parent_2_person_id"), reference=reference_by_household[household_rows], ) @@ -675,8 +711,28 @@ def _heads(rows: _Pointers, dependent: np.ndarray, rule: BenefitUnitRule) -> np. def _unit_rows( family: pd.DataFrame, membership: pd.Series, rule: BenefitUnitRule ) -> np.ndarray: - unit_ids = family[rule.id_column].to_numpy(dtype=np.int64) - rows = pd.Index(unit_ids).get_indexer(membership.to_numpy(dtype=np.int64)) + unit_ids = _int64_ids(family[rule.id_column], rule.id_column) + rows = pd.Index(unit_ids).get_indexer( + _int64_ids(membership, rule.membership_column) + ) if (rows < 0).any(): raise ValueError("A person's unit is missing from the unit table.") return rows + + +def _int64_ids(values: pd.Series, label: str) -> np.ndarray: + """Integer ids as int64, exactly; an id int64 cannot hold is refused. + + NumPy and pandas cast an unsigned id above ``2**63 - 1`` to a negative + one without complaint, so a bare cast could silently rename a person, + unit or household. + """ + + if values.dtype.kind not in "iu": + raise ValueError(f"{label} must hold integer ids, found dtype {values.dtype}.") + if _exceeds_int64(values): + raise ValueError( + f"{label} holds ids above {_INT64_MAX}; unit tables hold ids as " + "int64, which cannot represent them." + ) + return values.to_numpy(dtype=np.int64) diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py index 420ec55818..e35669c318 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_group_encode.py @@ -10,7 +10,11 @@ alternative conserves it too; - ``sum_over_members`` conserves member totals; ``any_member`` and ``reference_member`` read the members and the head; -- encoding is deterministic and row-aligned to the unit table. +- encoding is deterministic and row-aligned to the unit table; +- state bindings given as a list, iterator or generator encode, or are + refused, exactly as the same bindings given as a tuple; +- membership ids of another integer dtype encode exactly as int64 ones, or + are refused when they hold an id int64 cannot. A row-by-row reference written from each rule's documented meaning checks the vectorized executor (a differential test), and unit-composition flags are @@ -19,6 +23,7 @@ """ import json +import re import numpy as np import pandas as pd @@ -460,6 +465,51 @@ def test_knobs_zero_scale_and_redirect(self, case, factor) -> None: base.drop(columns=["earnings", "rent", "in_area_1"]), ) + @PROPERTY + @given( + case=unit_frames(), + picks=st.lists(st.integers(0, len(STATE) - 1), max_size=len(STATE) + 1), + knobs=st.sampled_from( + [ + GroupKnobs(), + GroupKnobs(state_columns={"household.area": "household.area_alt"}), + ] + ), + kind=st.sampled_from(["list", "iterator", "generator"]), + ) + def test_any_iterable_of_state_bindings_encodes_as_its_tuple( + self, case, picks, knobs, kind + ) -> None: + tables, rule, _, _, group = case + states = [STATE[pick] for pick in picks] + given_as = { + "list": list, + "iterator": iter, + "generator": lambda items: (item for item in items), + }[kind] + + def outcome(bindings): + try: + encoded = MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=bindings, knobs=knobs + ) + except ValueError as error: + return str(error) + return encoded + + expected = outcome(tuple(states)) + got = outcome(given_as(states)) + if isinstance(expected, str): + # a refusal (a binding given twice, or a knob no binding reads) + # is the same refusal + assert got == expected + return + assert not isinstance(got, str), got + pd.testing.assert_frame_equal( + got.tables[rule.entity], expected.tables[rule.entity] + ) + assert got.deferred == expected.deferred + @PROPERTY @given(case=unit_frames()) def test_encoding_is_deterministic(self, case) -> None: @@ -470,6 +520,27 @@ def test_encoding_is_deterministic(self, case) -> None: first.tables[rule.entity], second.tables[rule.entity] ) + def test_a_one_shot_iterator_of_state_bindings_takes_a_column_knob(self) -> None: + # The bindings are read twice: to encode, then to find the columns a + # knob may replace. A spent iterator used to leave the second read + # empty, so the area knob was refused as replacing a column no + # binding reads. + tables, rule, _, _, group = _example() + tables = {**tables, "household": tables["household"].assign(area_alt=2)} + knobs = GroupKnobs(state_columns={"household.area": "household.area_alt"}) + expected = MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=STATE, knobs=knobs + ) + assert expected.tables[rule.entity]["in_area_1"].tolist() == [False, False] + for states in (list(STATE), iter(STATE), (state for state in STATE)): + got = MAPPING.encode_groups( + tables, {"Family": group}, state_bindings=states, knobs=knobs + ) + pd.testing.assert_frame_equal( + got.tables[rule.entity], expected.tables[rule.entity] + ) + assert got.deferred == expected.deferred + def test_unexecutable_group_bindings_are_deferred(self) -> None: tables, _, _, _, group = _example() encoded = _encode(tables, group) @@ -763,6 +834,103 @@ def test_memberships_align_by_index_and_ids(self) -> None: with pytest.raises(ValueError, match="must be integers"): MAPPING.encode_groups(tables, {"Family": fractional}) + def test_a_membership_naming_an_unsigned_id_int64_cannot_hold_is_refused( + self, + ) -> None: + # Person 13 becomes -1, so the units are -1 and 10. As uint64, -1 is + # 2**64 - 1: a unit that does not exist, which an int64 comparison + # used to read back as unit -1 and accept. + tables, rule, _, _, _ = _example() + person = tables["person"].assign( + person_id=tables["person"]["person_id"].replace({13: -1}) + ) + tables = {**tables, "person": person} + family, membership = build_benefit_units(person, tables["household"], rule) + group = benefit_unit_membership(person, family, membership, rule) + assert membership.tolist() == [10, 10, 10, -1] + unsigned = GroupMembership( + entity="family", + units=family, + person_unit=membership.astype(np.uint64), + person_role=group.person_role, + ) + with pytest.raises(ValueError, match=re.escape("['person unit ids']")): + MAPPING.encode_groups(tables, {"Family": unsigned}) + # Likewise a frame whose person -1 is renamed 2**64 - 1: the unit's + # head column, -1, no longer names its head. + renamed = person.assign(person_id=person["person_id"].astype(np.uint64)) + with pytest.raises(ValueError, match=re.escape("['person.person_id']")): + MAPPING.encode_groups({**tables, "person": renamed}, {"Family": group}) + + def test_a_partner_pointer_int64_cannot_hold_is_refused(self) -> None: + # Head 10 becomes -1 and partner 11's pointer becomes 2**64 - 1, which + # names no person but an int64 read turns into -1: person 11 would pass + # as head -1's partner. An int64 pointer naming no person is refused as + # before, by the partner rule itself. + tables, _, family, _, group = _example() + person = tables["person"].copy() + for column in ("person_id", "parent_1_person_id", "parent_2_person_id"): + person[column] = person[column].replace({10: -1}) + household = tables["household"].assign( + reference_person_id=tables["household"]["reference_person_id"].replace( + {10: -1} + ) + ) + units = family.assign(family_id=[-1, 13], family_head_person_id=[-1, 13]) + membership = GroupMembership( + entity="family", + units=units, + person_unit=pd.Series([-1, -1, -1, 13], index=person.index, dtype=np.int64), + person_role=group.person_role, + ) + wrapped = person.assign( + partner_person_id=pd.array([11, 2**64 - 1, None, None], dtype="UInt64") + ) + with pytest.raises(ValueError, match=re.escape("['person.partner_person_id']")): + MAPPING.encode_groups( + {**tables, "person": wrapped, "household": household}, + {"Family": membership}, + ) + dangling = person.assign( + partner_person_id=pd.array([11, 999, None, None], dtype="Int64") + ) + with pytest.raises(ValueError, match="must be its head's partner"): + MAPPING.encode_groups( + {**tables, "person": dangling, "household": household}, + {"Family": membership}, + ) + + @PROPERTY + @given(case=unit_frames(), dtype=st.sampled_from(["uint64", "UInt64", "Int64"])) + def test_membership_ids_are_read_exactly_or_refused(self, case, dtype) -> None: + tables, rule, family, membership, group = case + columns = (rule.id_column, rule.household_column, rule.head_column) + + def retyped(values: pd.Series) -> pd.Series: + # NumPy's cast turns a negative id into one above int64's range. + unsigned = dtype.lower().startswith("u") + raw = values.to_numpy().astype(np.uint64 if unsigned else np.int64) + return pd.Series(raw, index=values.index, name=values.name).astype(dtype) + + units = family.assign(**{column: retyped(family[column]) for column in columns}) + retyped_group = GroupMembership( + entity=group.entity, + units=units, + person_unit=retyped(membership), + person_role=group.person_role, + ) + ids = [value for column in columns for value in family[column].tolist()] + if dtype.lower().startswith("u") and min(ids) < 0: + with pytest.raises(ValueError, match="int64"): + _encode(tables, retyped_group) + return + expected = _encode(tables, group).tables[rule.entity] + got = _encode(tables, retyped_group).tables[rule.entity] + assert got[rule.id_column].tolist() == expected[rule.id_column].tolist() + pd.testing.assert_frame_equal( + got.drop(columns=rule.id_column), expected.drop(columns=rule.id_column) + ) + def test_roles_must_agree_with_the_partner_pointers(self) -> None: tables, _, family, membership, group = _example() # The couple's head is 10; claim the child 12 as the head's partner diff --git a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py index fc8142bb8c..943a53b3c9 100644 --- a/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py +++ b/packages/microcosm-frame/tests/engine_free/shared/test_unit_construction.py @@ -13,10 +13,14 @@ - units nest in households; - family weights are inherited: the sum of family weights equals the sum of household weight times units per household; -- unit ids are deterministic and independent of row order. +- unit ids are deterministic and independent of row order; +- ids of every integer dtype the contract accepts are kept exactly, or + refused when ``int64`` cannot hold them, and the dtype changes no unit. """ import json +import re +from collections import Counter import numpy as np import pandas as pd @@ -26,6 +30,7 @@ from microcosm.frame import EntitySchema, Frame, WeightKind, Weights from microcosm.frame.concept_mapping import UnitRole +from microcosm.frame.concepts import validate_concept_tables from microcosm.frame.unit_construction import ( BENEFIT_UNIT_RULE_FORMAT, BenefitUnitRule, @@ -46,6 +51,83 @@ #: Ids may be negative: the concept-frame contract accepts any unique integer, #: so no unit reading may treat an id such as -1 as "absent". FRAMES = concept_frames(max_households=5, max_members=6, min_id=-(10**9)) +POINTERS = ("partner_person_id", "parent_1_person_id", "parent_2_person_id") +INT64_MAX = int(np.iinfo(np.int64).max) +#: Every integer dtype the concept-frame contract accepts for an id column; +#: pointers take the matching nullable dtype. +ID_DTYPES = ("int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64") + + +def _nullable(dtype: str) -> str: + return "U" + dtype[1:].capitalize() if dtype.startswith("u") else dtype.capitalize() + + +@st.composite +def retyped_frames(draw) -> dict[str, pd.DataFrame]: + """A valid frame whose ids and pointers use accepted integer dtypes. + + Person ids (and the pointers that name them) take one dtype and household + ids another, so signed and unsigned tables are mixed; ids come from each + dtype's whole range, with its edges and int64's edge mixed in. + """ + + tables = draw(concept_frames(max_households=4, max_members=4)) + person, household = tables["person"].copy(), tables["household"].copy() + person_dtype = draw(st.sampled_from(ID_DTYPES)) + household_dtype = draw(st.sampled_from(ID_DTYPES)) + + def renamed(old: list[int], dtype: str) -> dict[int, int]: + info = np.iinfo(dtype) + low, high = int(info.min), int(info.max) + edges = [ + value + for value in (low, high, 0, -1, INT64_MAX, INT64_MAX + 1) + if low <= value <= high + ] + ids = st.integers(low, high) | st.sampled_from(edges) + new = draw(st.lists(ids, min_size=len(old), max_size=len(old), unique=True)) + return dict(zip(old, new, strict=True)) + + persons = renamed(person["person_id"].tolist(), person_dtype) + households = renamed(household["household_id"].tolist(), household_dtype) + nullable = _nullable(person_dtype) + + def ids(values: pd.Series, names: dict[int, int], dtype: str) -> np.ndarray: + return np.array([names[value] for value in values.tolist()], dtype=dtype) + + def pointers(values: pd.Series) -> pd.api.extensions.ExtensionArray: + return pd.array( + [None if pd.isna(value) else persons[int(value)] for value in values], + dtype=nullable, + ) + + person["person_id"] = ids(person["person_id"], persons, person_dtype) + person["person_household_id"] = ids( + person["person_household_id"], households, household_dtype + ) + for column in POINTERS: + person[column] = pointers(person[column]) + household["household_id"] = ids( + household["household_id"], households, household_dtype + ) + household["reference_person_id"] = pointers(household["reference_person_id"]) + return {"person": person, "household": household} + + +def _as_int64(person: pd.DataFrame, household: pd.DataFrame): + """The same frame with int64 ids and Int64 pointers.""" + + person = person.astype( + { + "person_id": np.int64, + "person_household_id": np.int64, + **dict.fromkeys(POINTERS, "Int64"), + } + ) + household = household.astype( + {"household_id": np.int64, "reference_person_id": "Int64"} + ) + return person, household def _rows(person: pd.DataFrame, column: str) -> np.ndarray: @@ -261,6 +343,53 @@ def test_attributes_count_the_roles(self, tables, rule) -> None: assert membership_view.entity == rule.entity pd.testing.assert_series_equal(membership_view.person_role, roles) + @PROPERTY + @given(tables=retyped_frames(), rule=unit_rules()) + def test_ids_of_every_accepted_dtype_are_kept_exactly_or_refused( + self, tables, rule + ) -> None: + person, household = tables["person"], tables["household"] + # Python integers throughout, so no cast can make two ids agree. + person_ids = person["person_id"].tolist() + declared = dict( + zip(person_ids, person["person_household_id"].tolist(), strict=True) + ) + household_ids = household["household_id"].tolist() + if max(person_ids + household_ids) > INT64_MAX: + with pytest.raises(ValueError, match="int64"): + build_benefit_units(person, household, rule) + return + family, membership = build_benefit_units(person, household, rule) + # identity: a unit's id is its head's own person id, and it nests in + # the household its head declares + unit_ids = family[rule.id_column].tolist() + unit_households = family[rule.household_column].tolist() + assert unit_ids == family[rule.head_column].tolist() + assert unit_households == [declared[head] for head in unit_ids] + # nesting: each person's unit lies in the person's declared household + household_of = dict(zip(unit_ids, unit_households, strict=True)) + assert [household_of[unit] for unit in membership.tolist()] == [ + declared[person_id] for person_id in person_ids + ] + # unit counts: every unit counted once, in its declared household + counts = units_per_household(household, family, rule) + expected = Counter(unit_households) + assert counts.tolist() == [expected[value] for value in household_ids] + assert int(counts.sum()) == len(family) + # the dtype changes nothing: the same ids as int64 build the same units + person_64, household_64 = _as_int64(person, household) + family_64, membership_64 = build_benefit_units(person_64, household_64, rule) + pd.testing.assert_frame_equal(family, family_64) + pd.testing.assert_series_equal(membership, membership_64) + pd.testing.assert_series_equal( + benefit_unit_roles(person, family, membership, rule), + benefit_unit_roles(person_64, family_64, membership_64, rule), + ) + pd.testing.assert_frame_equal( + benefit_unit_attributes(person, family, membership, rule), + benefit_unit_attributes(person_64, family_64, membership_64, rule), + ) + class TestPlacement: RULE = BenefitUnitRule( @@ -466,6 +595,100 @@ def _fields(self) -> dict[str, object]: } +class TestIdRange: + """Unit tables hold int64 ids; the contract also accepts unsigned ids.""" + + RULE = TestPlacement.RULE + + @staticmethod + def _one_adult(household_id: int) -> tuple[pd.DataFrame, pd.DataFrame]: + person = pd.DataFrame( + { + "person_id": np.array([1], dtype=np.int64), + "person_household_id": np.array([household_id], dtype=np.uint64), + "age": np.array([30], dtype=np.int64), + **{column: pd.array([None], dtype="Int64") for column in POINTERS}, + } + ) + household = pd.DataFrame( + { + "household_id": np.array([household_id], dtype=np.uint64), + "reference_person_id": pd.array([1], dtype="Int64"), + } + ) + return person, household + + def test_the_largest_unsigned_id_int64_holds_is_kept_exactly(self) -> None: + person, household = self._one_adult(INT64_MAX) + family, membership = build_benefit_units(person, household, self.RULE) + assert family.to_dict("list") == { + "family_id": [1], + "family_household_id": [INT64_MAX], + "family_head_person_id": [1], + } + assert membership.tolist() == [1] + assert units_per_household(household, family, self.RULE).tolist() == [1] + + @pytest.mark.parametrize("household_id", [INT64_MAX + 1, 2**64 - 1]) + def test_an_unsigned_household_id_int64_cannot_hold_is_refused( + self, household_id + ) -> None: + # The concept-frame contract accepts this frame. Cast to int64, 2**63 + # became household -2**63: the unit nested in no household and its + # household counted no unit. + person, household = self._one_adult(household_id) + assert not validate_concept_tables({"person": person, "household": household}) + columns = "['person.person_household_id', 'household.household_id']" + with pytest.raises(ValueError, match=re.escape(columns)): + build_benefit_units(person, household, self.RULE) + + def test_an_unsigned_pointer_int64_cannot_hold_is_refused(self) -> None: + # 2**64 - 1 names no person here, but cast to int64 it is -1, who is + # a person here: the pair would have passed as partners. + person = pd.DataFrame( + { + "person_id": np.array([-1, 5], dtype=np.int64), + "person_household_id": np.array([1, 1], dtype=np.int64), + "age": np.array([40, 40], dtype=np.int64), + "partner_person_id": pd.array([5, 2**64 - 1], dtype="UInt64"), + "parent_1_person_id": pd.array([None, None], dtype="Int64"), + "parent_2_person_id": pd.array([None, None], dtype="Int64"), + } + ) + household = pd.DataFrame( + { + "household_id": np.array([1], dtype=np.int64), + "reference_person_id": pd.array([5], dtype="Int64"), + } + ) + with pytest.raises(ValueError, match=re.escape("['person.partner_person_id']")): + build_benefit_units(person, household, self.RULE) + + def test_unit_readers_refuse_ids_int64_cannot_hold(self) -> None: + person, household = TestPlacement._frame( + [(-1, 1, 40, None, None, None, 0.0), (5, 1, 40, None, None, None, 0.0)], + {1: 5}, + ) + family, membership = build_benefit_units(person, household, self.RULE) + assert membership.tolist() == [-1, 5] + # Unit 2**64 - 1 does not exist; cast to int64 it would be unit -1. + wrapped = pd.Series( + np.array([2**64 - 1, 5], dtype=np.uint64), + index=membership.index, + name=membership.name, + ) + for read in ( + benefit_unit_roles, + benefit_unit_attributes, + benefit_unit_membership, + ): + with pytest.raises(ValueError, match="int64"): + read(person, family, wrapped, self.RULE) + # A float id is refused too: a cast would truncate it. + with pytest.raises(ValueError, match="integer ids"): + benefit_unit_roles(person, family, membership.astype(float), self.RULE) + + class TestRuleSerialization: @PROPERTY @given(rule=unit_rules(refusing=True))