diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv index b404e7b9..d79f702d 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv @@ -2747,49 +2747,49 @@ us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_er us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly denied the listed alimony deduction and used an estimated $10,207 post-TCJA-reversion standard deduction. PolicyEngine deducts the $1,165 alimony expense and applies the applicable 2026 single age-65-or-older standard deduction, leaving no taxable income." us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced zero taxable income, zero tax, and zero nonrefundable credits, but it submitted $1,268.50. Its numeric output directly contradicts every computation step in its explanation." us,scenario_031,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $5,357.50 of Social Security as taxable even though provisional income is below the $25,000 single-filer base, and it then failed to follow its own calculation that the elderly credit reduced tax to zero. The alimony adjustment yields $13,243 of AGI, the standard deduction eliminates taxable income, and the submitted $2,439 has no support in its stated arithmetic." -us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It correctly picked the aged pathway and its 138% FPL limit. But it built countable income by subtracting only alimony ($22,688) and never subtracted the Medicare Part B premium (about $2,435/year). With that deduction, countable income is about $21,400, which is under the limit." -us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It applied only the $240 general income exclusion ($23,613) against the 138% FPL aged/disabled limit. It left out the senior-or-disabled deduction for health insurance premiums, including Medicare Part B (about $2,435/year), which puts countable income below the roughly $21,600 limit." -us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with a 138% FPL figure understated at $18,564 and never deducted the Medicare Part B premium from countable income. With Part B subtracted, countable income is about $21,400, under 138% FPL (about $21,600). It also doubted the $4,200 in assets, although that amount easily passes California's senior-or-disabled asset test." -us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It changed its answer to a 100% FPL limit (about $15,650). California's senior-or-disabled pathway uses 138% FPL. It also counted the full $23,853 without deducting the Medicare Part B premium, which brings countable income to about $21,400, within the limit." -us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It tested gross income of $23,853 against a 100–138% FPL range and never subtracted health insurance premiums. The non-MAGI senior-or-disabled count deducts the Medicare Part B premium (about $2,435/year), which puts income under the 138% FPL limit." -us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the correct 138% FPL aged limit but compared it with gross income of $23,853. Under this pathway, countable income is net of the Medicare Part B premium (about $2,435/year), leaving about $21,400, which is below the limit." -us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It switched between the ACA expansion test, which does not cover someone aged 67, and a 100% FPL aged limit. It settled on gross $23,853 against an understated 138% FPL figure of $20,783. It never used the senior-or-disabled pathway's deduction of the Medicare Part B premium, which brings countable income under the real 138% FPL limit." -us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It first stated the correct Aged & Disabled FPL limit (about $21,597), which its own $14,408 figure would pass. Then it switched to the SSI federal benefit rate (about $13,596) as the binding limit. It also treated Social Security as MAGI-excluded, although the non-MAGI count includes Social Security in full and instead deducts the Medicare Part B premium." -us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It said the head is in no categorical group, but at 67 the head is aged, which is exactly the senior-or-disabled category. It also used unadjusted income of about $23.9k and skipped the Medicare Part B premium deduction, which brings countable income below 138% FPL." -us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with 138% FPL without deducting health insurance premiums. The senior-or-disabled pathway subtracts the Medicare Part B premium (about $2,435/year), leaving countable income of about $21,400, under the limit." -us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used gross monthly income of $1,987.75 against an understated 138% FPL of $1,732/month. It never deducted the Medicare Part B premium (about $203/month), which brings countable income to about $1,785/month, under the correct 138% FPL of about $1,800/month." -us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It treated the full $23,853 as countable income and required SSI receipt for a categorical route. The senior-or-disabled pathway does not require SSI and deducts the Medicare Part B premium, leaving countable income below 138% FPL." -us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with its own 138% FPL figure of $22,218. Deducting the Medicare Part B premium (about $2,435/year), as the senior-or-disabled income count requires, leaves about $21,400, below even the model's own threshold." -us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It picked the correct 138% FPL Aged & Disabled limit (about $22,080 by its estimate) but tested gross income of $23,853. The pathway deducts Medicare Part B premiums from countable income, which brings it to about $21,400, under the limit." -us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no derivation, and its 'No' is consistent with comparing gross income of $23,853 with an FPL limit. The senior-or-disabled pathway subtracts the Medicare Part B premium, which leaves countable income of about $21,400, below 138% FPL, so the head is eligible." -us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It tested gross income of $23,853 against the 138% FPL Aged & Disabled limit without deducting health insurance premiums. Subtracting the Medicare Part B premium (about $2,435/year) puts countable income under the limit." -us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It correctly identified the non-MAGI 138% FPL limit but measured gross monthly income against it. The non-MAGI senior-or-disabled count deducts the Medicare Part B premium, which brings income below the limit." -us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no derivation. The correct path is the aged (senior-or-disabled) category: $23,853 gross minus the Medicare Part B premium gives about $21,400, under 138% FPL, with assets passing. The 'No' reflects skipping that premium deduction." -us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It compared the full $23,853 with the aged Medi-Cal threshold without subtracting the Medicare Part B premium. The senior-or-disabled pathway deducts that premium, leaving countable income below 138% FPL." -us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It correctly identified the non-MAGI elderly pathway but did not deduct health insurance premiums, including Medicare Part B, from countable income. That deduction brings income from $23,853 to about $21,400, under the 138% FPL limit." -us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It said total income 'well' exceeds 138% FPL, but gross income is only about 150% FPL. After the senior-or-disabled pathway deducts the Medicare Part B premium, countable income falls below 138% FPL." -us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed Medicare eligibility rules out Medicaid. Medicare-eligible seniors qualify for Medicaid (dual eligibility) through the aged senior-or-disabled pathway, which this head meets on income after the Part B deduction and on assets." -us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It tested the head against the ACA adult expansion limit, which does not apply at age 67, and overstated income as about 199% FPL; $23,853 is about 150% FPL. The applicable pathway is senior-or-disabled, where countable income after the Medicare Part B premium deduction is under 138% FPL." -us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It looked for a disability flag and missed that age 67 alone meets the aged criterion for the senior-or-disabled pathway. Under that pathway, income net of the Medicare Part B premium is below 138% FPL and the $4,200 in assets passes the asset test." -us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied a spend-down (medically needy) standard and counted only the $50 over-the-counter expense as a medical deduction. The applicable pathway is the 138% FPL senior-or-disabled program, which deducts the Medicare Part B premium (about $2,435/year) and leaves countable income under the limit." -us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no computation, and its 'above threshold' conclusion matches testing gross income. The senior-or-disabled pathway subtracts the Medicare Part B premium from $23,853, leaving about $21,400, below 138% FPL." -us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It correctly picked the aged pathway but compared unadjusted retirement income with the limit. That pathway deducts health insurance premiums, including Medicare Part B, which brings countable income under 138% FPL." -us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It used an SSI-linked income level as the aged Medicaid test. California's optional senior-or-disabled pathway uses 138% FPL, and after the Medicare Part B premium deduction the head's countable income is under that limit." -us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It applied only the $20/month general income exclusion and left out the senior-or-disabled deduction for health insurance premiums. Subtracting the Medicare Part B premium (about $2,435/year) brings countable income below 138% FPL." -us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It compared gross retirement income with the aged threshold without deducting the Medicare Part B premium. The senior-or-disabled income count requires that deduction, which leaves the head at about $21,400, under 138% FPL." -us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It deducted only the general income exclusion and missed the health insurance premium deduction, including Medicare Part B, that the senior-or-disabled pathway applies. With that deduction, countable income falls below 138% FPL." -us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It claimed PolicyEngine excludes Medicare-eligible (dual-eligible) people from Medicaid. The aged senior-or-disabled pathway is built for Medicare-age individuals, and this head meets its income test (after the Part B deduction) and its asset test." -us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It treated $4,200 in bank assets as over the aged resource limit, but $4,200 easily passes California's senior-or-disabled asset test ($130,000 for an individual in 2026). It also tested gross income of $23,853 without deducting the Medicare Part B premium, which brings countable income under 138% FPL." -us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"It mixed MAGI and ABD counting and compared the full $23,853 with the Aged & Disabled FPL limit. The non-MAGI count deducts the Medicare Part B premium (about $2,435/year), leaving about $21,400, under 138% FPL." -us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied only the $20 monthly unearned-income disregard before testing against the 138% FPL Aged & Disabled limit. It missed the deduction of health insurance premiums, including Medicare Part B, which brings countable income below the limit." -us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied the $2,000 SSI resource limit. California's senior-or-disabled Medi-Cal asset limit is $130,000 for an individual in 2026, so $4,200 passes, and income after the Medicare Part B deduction is under 138% FPL." -us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It used a 100% FPL limit (about $15,650) for aged Medi-Cal, but California's senior-or-disabled pathway uses 138% FPL. It also counted gross income without deducting the Medicare Part B premium, which brings countable income to about $21,400, under the limit." -us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It said the head fits no non-MAGI aged or disabled pathway because they are not disabled or on SSI. At 67 the head is aged, which qualifies them for the senior-or-disabled category without SSI receipt, and they pass its income test (after the Part B deduction) and its asset test." -us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly excluded the adult expansion group but compared unadjusted retirement income with the aged limit. The senior-or-disabled pathway deducts the Medicare Part B premium, which leaves countable income under 138% FPL." -us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It guessed that roughly $24k of income exceeds the senior limit without using any threshold or deduction. The correct calculation subtracts the Medicare Part B premium (about $2,435) from $23,853, which gives about $21,400, under 138% FPL." -us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It applied the SSI $2,000 resource limit to the $4,200 in bank assets. California's senior-or-disabled asset test ($130,000 individual limit in 2026) is passed. It also compared gross income with the income standard without deducting the Medicare Part B premium, which brings income below 138% FPL." -us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with 'typical' ABD limits and treated missing SSI or disability status as disqualifying. The aged senior-or-disabled pathway requires neither, and it deducts the Medicare Part B premium, which leaves countable income under 138% FPL." -us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It concluded the 67-year-old is in no eligibility category. Age 65+ is itself the aged criterion for the senior-or-disabled pathway, where income net of the Medicare Part B premium is below 138% FPL and assets pass." +us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It took the aged pathway and a 138% limit of about $1,800 a month, subtracted the $1,165 of alimony to put income at about $1,890 a month, and assumed no specific disregard, saying even SSI-style disregards would leave it over. The engine does not subtract alimony; it takes $1,987.79 a month less California's $230 disregard, which replaces SSI's $20 exclusion, for $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,claude-fable-5.1,llm_error,taxable_income_or_deductions,False,"It used the aged pathway but applied only SSI's $240 annual general exclusion, putting countable income at $23,613 against a 138% limit of about $21,600. California's $230 monthly disregard takes the place of that $20 exclusion, so the engine counts $23,853.47 less $2,760, or $21,093.47, which is under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853, and that amount less the $1,165 of alimony, against a 138% limit it understated at about $18,564, and doubted the $4,200 in assets. The engine subtracts no alimony: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline). The $4,200 is far under California's $130,000 asset limit." +us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"It first called gross income of $23,853 well below the threshold, then switched to a MAGI-based 100% limit of about $15,650 (the 2025 guideline), and its explanation concluded the head was ineligible. The engine's senior pathway uses SSI income rules and 138% of the 2026 guideline, $22,024.80; it counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the limit." +us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It tested gross income of $23,853 against an aged limit it gave as 100% to 138% of the poverty line, about $15,650 to $21,600 (2025-guideline figures), and found it over. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,claude-opus-5.5,llm_error,taxable_income_or_deductions,False,"It used the 138% aged limit, at about $21,600 (the 2025-guideline figure), but compared it with gross income of $23,853. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under its $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It moved between the adult expansion test, which excludes someone aged 67, and a 100% aged limit of $15,060; its explanation settled on the expansion test, with gross income of $23,853 against $20,783, which is 138% of its understated $15,060 guideline. The engine's senior pathway counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under $22,024.80 (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"It excluded most Social Security to get $14,408, said 138% applied only to expansion adults under 65, cited a 138% aged limit of about $21,597 that $14,408 would pass, then used an SSI limit of about $13,596 as binding. The engine counts Social Security in full and uses 138% of the 2026 guideline, $22,024.80: $23,853.47 less California's $230 monthly disregard ($2,760 a year) gives $21,093.47, under the limit." +us,scenario_031,head_medicaid_eligible,claude-sonnet-5.5,llm_error,categorical_eligibility,False,"It said the head was in no categorical group and that countable income of about $23.9k, including Social Security and with no disregard, exceeded 138% FPL. At 67 the head is aged, which alone meets the category condition of the engine's senior-or-disabled pathway, and the engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,deepseek-v4-flash-0731,llm_error,taxable_income_or_deductions,False,"It compared gross income of about $23,853 with 138% FPL and concluded the head was over the limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It tested gross monthly income of $1,987.75 against an understated 138% FPL limit of $1,732. The engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $1,757.79, under the actual limit of $1,835.40 a month (138% of the 2026 guideline). Both errors mattered: income after the disregard still tops the model's $1,732 figure." +us,scenario_031,head_medicaid_eligible,deepseek-v4-pro-0813,llm_error,taxable_income_or_deductions,False,"It treated the full $23,853 as countable income and found it over California's FPL-based limit for a senior, without stating the limit. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,deepseek-v4.1-flash,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with its own 138% FPL figure of $22,218, slightly above the engine's $22,024.80 limit (138% of the 2026 guideline). The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under either figure." +us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It used a 138% FPL limit of about $22,080, close to the engine's $22,024.80 (138% of the 2026 guideline), but tested gross income of $23,853 against it. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves countable income of $21,093.47, under the limit." +us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It gave no figures or pathway, saying only that household income exceeded the Medicaid limit ""for a single adult."" At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It compared gross income of $23,853 with 138% FPL, without stating the dollar limit, and found it over. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It correctly picked the 138% FPL limit for what it called ""free non-MAGI Medi-Cal"" but gave no figures, saying only that the head's monthly income was over it; in the same response it put gross income at $23,853 for its federal tax and SNAP answers. The engine subtracts California's $230 monthly disregard from $1,987.79 a month, in place of SSI's $20 exclusion. That leaves $1,757.79 a month, under the $1,835.40 monthly limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gemini-3.5-flash-lite,llm_error,taxable_income_or_deductions,False,"It gave no figures, threshold or pathway, citing only the head's age and income. At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) is $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It treated the full $23,853 as countable and said it exceeded an unstated threshold for people 65 or older. Gross income is $1,828.67 over the limit, but the engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It placed the head in what it called the ""non-MAGI"" elderly pathway but gave no income figure or limit, and it judged income too high, which holds against the engine's limit only if California's $230 disregard is skipped or replaced by a smaller one, such as SSI's $20. In the engine's senior-or-disabled pathway, $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gemini-3.8-flash,llm_error,taxable_income_or_deductions,False,"It tested total income against a 138% FPL limit with no disregard and called it ""well"" over, but $23,853.47 is about 149% of the 2026 guideline, only $1,828.67 over the $22,024.80 limit. The engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the limit." +us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"It assumed that Medicare eligibility at 65 or older bars Medicaid and ran no income test. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group, and age 67 alone meets the aged test for the senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,glm-5.3,llm_error,categorical_eligibility,False,"It applied the adult expansion limit, which excludes anyone over 64, comparing gross income of about $1,988 a month (about 149% of the 2026 guideline, not the 199% it stated) with an understated limit of about $1,600. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $1,987.79 less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It denied eligibility because no disability or low-income flag was listed and income was ""substantial"", without comparing any income figure with a limit. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It correctly ruled out the adult expansion group at age 67 but tested the full $23,853 against an unstated aged income or ""spend-down"" standard, with only minimal medical expenses to offset it. The engine's senior-or-disabled pathway instead subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It gave no figures, pathway or threshold and only asserted that countable income was above the applicable California limit, which holds against the engine's limit only if the state's $230 monthly disregard ($2,760 a year) is skipped or replaced by a smaller one, such as SSI's $20. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $23,853.47 less that disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It identified the aged pathway but concluded, without showing figures, that retirement income exceeded California's limit. That holds against the engine's limit only if California's $230 monthly disregard is skipped or replaced by a smaller one, such as SSI's $20: gross income of $23,853.47 is $1,828.67 over the limit. The engine's senior-or-disabled pathway subtracts that disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"It placed the head in the aged pathway but tested income against an ""SSI-linked income level,"" giving no figures. The engine's limit for this pathway in California is $22,024.80 (138% of the 2026 poverty guideline), and countable income is $21,093.47: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion." +us,scenario_031,head_medicaid_eligible,gpt-6-astra,llm_error,taxable_income_or_deductions,False,"It correctly ruled out the adult expansion group at 67 but said retirement income after the general income exclusion exceeded California's aged limit, giving no figures. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline)." +us,scenario_031,head_medicaid_eligible,gpt-6-sol,llm_error,taxable_income_or_deductions,False,"It said the head's retirement income exceeded California's aged Medicaid threshold, naming no exclusion and giving no figures. Gross income of $23,853.47 is over the $22,024.80 limit (138% of the 2026 poverty guideline), but the engine first subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving countable income of $21,093.47, under the limit." +us,scenario_031,head_medicaid_eligible,gpt-6.1-sol,llm_error,taxable_income_or_deductions,False,"It said retirement income exceeded California's aged limit after ""the applicable general income exclusion,"" without stating its amount, the income or the limit. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline)." +us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"It ruled the head out as Medicare-eligible, claiming the engine excludes ""full dual eligibles"" from Medicaid. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group: at 67 the head counts as aged for the senior-or-disabled pathway, countable income of $21,093.47 ($23,853.47 less California's $230 monthly disregard, $2,760 a year) is under the $22,024.80 limit (138% of the 2026 poverty guideline), and $4,200 in assets is under the $130,000 limit." +us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"It said bank assets of $4,200 and countable income of about $23,853 both exceeded California's aged limits, counting gross income in full. The pathway's asset limit is $130,000, so $4,200 passes, and the engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline)." +us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"It counted the full $23,853 as ""MAGI/ABD countable income"" and said it exceeded California's ""Aged & Disabled FPL"" limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline)." +us,scenario_031,head_medicaid_eligible,grok-4.7,llm_error,taxable_income_or_deductions,False,"It applied only the $20 monthly unearned-income disregard before testing against California's 138% FPL limit, which leaves about $23,613, still over the $22,024.80 limit. The engine uses California's $230 monthly disregard in place of that $20 exclusion, so countable income is $21,093.47 ($23,853.47 less $2,760), under the limit." +us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"It applied a $2,000 asset limit to aged Medicaid and denied eligibility because the $4,200 in bank assets exceeded it, but $2,000 is SSI's resource limit, not this pathway's test. The engine's California senior-or-disabled pathway allows assets below $130,000, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"It applied a 100% FPL limit of about $15,650, the 2025 guideline, and compared gross income of about $23,853 against it. The engine uses 138% of the 2026 guideline, $22,024.80, and subtracts California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) from $23,853.47, leaving countable income of $21,093.47, under the limit." +us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"It correctly placed the head outside the adult expansion group but ruled out the aged and disabled pathways because the head was not disabled, blind or an SSI recipient. The engine's senior-or-disabled pathway needs no disability, blindness or SSI receipt; age 67 alone meets its aged criterion. Countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly placed the head outside the adult expansion group but said the listed retirement income exceeded the California aged income limit, without naming a limit or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"It guessed that about $24,000 of income might exceed an unstated senior limit and answered 0 without naming a threshold or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving countable income of $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"It correctly treated the head as aged but applied SSI's $2,000 resource limit to the $4,200 in bank assets and said gross income of about $24,000 exceeded the aged income standard. The engine's California pathway allows assets below $130,000 and subtracts a $230 monthly disregard ($2,760 a year) from the $23,853.47 of income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It counted $23,853 of Social Security, pension and IRA income in full, said it exceeded ""typical aged/blind/disabled Medicaid limits"" without naming one, and cited the missing disability or SSI status. The engine's senior-or-disabled pathway needs neither, since age 67 alone meets its aged criterion, and it subtracts California's $230 monthly disregard ($2,760 a year) from $23,853.47, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." +us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"It said the 67-year-old was not disabled, blind or in any other listed category, and that income was above the senior pathway's limit without naming a figure. Age 65 or older alone meets the aged criterion of the engine's senior-or-disabled pathway, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline)." us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated gross income under 200% FPL as automatic BBCE eligibility and never ran the net income test PolicyEngine applies to this elderly household. Net income of about $1,779/mo ($1,987.75 minus the $209 standard deduction, with no shelter or medical deduction) is above the roughly $1,305/mo net limit. The household is ineligible, so the $23/mo minimum benefit ($276/yr) does not apply." us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model assumed BBCE at 200% FPL makes a 1-person household categorically eligible and entitled to the minimum benefit. It skipped the 100% FPL net income test that PolicyEngine applies to this elderly household. Net income of about $1,779/mo is above the roughly $1,305/mo limit, so the household is ineligible and gets $0, not $276." us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated countable income as very low. In fact, all $23,853 of Social Security, pension and IRA income counts as unearned income, which is $1,987.75/mo gross. Its $144/mo ($1,728/yr) benefit only works if net income is about $513/mo, which means it left out most of the pension and Social Security income. The real net income of about $1,779/mo fails the net income test, and 30% of it ($534) is more than the $298 maximum allotment, so SNAP is $0." diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv index e8afbc5b..9be4c099 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_notes.csv @@ -275,7 +275,7 @@ us,scenario_030,snap,46,llm_error,categorical_eligibility,False,,"The SNAP unit us,scenario_030,ssi,1,parse_contract_failure,missing_output,False,,All wrong responses were missing or unparseable predictions. us,scenario_030,tanf,2,llm_error,categorical_eligibility,False,,Texas TANF requires a family with a dependent child; pregnancy alone does not turn the unborn child into a qualifying dependent child or create a two-person assistance unit for this annual TANF calculation. The single adult therefore has no qualifying child and receives zero TANF. us,scenario_031,federal_income_tax_before_refundable_credits,4,llm_error,taxable_income_or_deductions,False,,"The $1,165 alimony expense reduces $14,408 of IRA and pension income to $13,243 of adjusted gross income, while Social Security remains nontaxable. The applicable single-filer standard deduction, including the age-65 addition, exceeds that income, producing zero taxable income and therefore zero federal income tax before refundable credits." -us,scenario_031,head_medicaid_eligible,43,llm_error,taxable_income_or_deductions,False,,"At 67, the head qualifies as aged, so the engine tests Medi-Cal through the optional senior-or-disabled pathway (California's Aged & Disabled FPL program). This is a non-MAGI pathway with a 138% FPL income limit. Countable income is gross income minus health insurance premiums, including the Medicare Part B standard premium (about $202.90/month, or about $2,435/year in 2026). That brings the $23,853 gross down to about $21,400, below 138% FPL (about $21,600). The $4,200 in bank assets easily passes California's asset test. Almost every model compared gross or near-gross income with the limit and never subtracted the Part B premium. Others used the wrong threshold (100% FPL or the SSI rate), applied the $2,000 SSI resource limit, or denied that an aged person has a Medicaid category." +us,scenario_031,head_medicaid_eligible,43,llm_error,taxable_income_or_deductions,False,,"At 67 the head is aged (65 or older), which alone meets the category condition of the engine's optional senior-or-disabled pathway; it needs no disability or SSI receipt. The adult expansion group (ages 19 to 64, no Medicare eligibility) does not apply, but in the engine's Medicaid rules Medicare eligibility bars only that group. The pathway counts income under SSI rules: Social Security ($9,445.47, in full), the $13,608 pension and the $800 IRA distribution give $23,853.47; the $1,165 of alimony paid is not subtracted. In place of SSI's $20 monthly general income exclusion, the engine applies California's $230 monthly disregard ($2,760 a year), leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline of $15,960). The $4,200 in bank assets is far below California's $130,000 asset limit for the pathway. The engine also models a $2,434.80 Medicare Part B premium for the head, but countable income does not subtract it: with the premium at zero, or the head not enrolled in Medicare, countable income is still $21,093.47. Most models (29 of 43) miscounted income: most of those compared about $23,853 with the limit, a few applied only SSI's $20 exclusion or an exclusion of unstated size, and several asserted without figures that income was too high. The rest used the wrong threshold (100% of the poverty guideline or an SSI income limit), applied SSI's $2,000 resource limit or otherwise judged the $4,200 too much, or denied the head a category, by treating Medicare eligibility or the lack of SSI receipt or a disability as disqualifying, or by testing only the adult expansion group." us,scenario_031,snap,6,llm_error,categorical_eligibility,False,,"The head is 67, so this is an elderly household. PolicyEngine does not let California's 200% FPL broad-based categorical eligibility skip the net income test for this household. Monthly gross income is $1,987.75 (Social Security, pension and IRA). The only deduction is the $209 standard deduction. There are no shelter costs, and the $50/yr OTC medical expense is below the $35/mo threshold, so net income is about $1,779/mo. That is well above the 1-person net limit of about $1,305/mo (100% FPL), so the household is not SNAP-eligible. The one/two-person minimum benefit only applies to eligible households, so SNAP is $0. Five models treated BBCE as automatic eligibility and paid the minimum benefit. The sixth undercounted income." us,scenario_031,state_income_tax_before_refundable_credits,1,llm_error,credit_phaseout,False,,"California first computes $75.37 of tax on $7,537 of taxable income, then applies $312.93 of nonrefundable credits, including the aged-or-blind exemption credit. Those credits fully eliminate the liability, so state income tax after nonrefundable credits and before refundable credits is $0." us,scenario_032,child1_chip_eligible,13,llm_error,health_coverage,False,,"CHIP is residual by statute: 42 U.S.C. 1397jj(b)(1)(A) and 42 CFR 457.310 exclude from ""targeted low-income child"" any child who is eligible for Medicaid, so the CHIP question is settled by first running the Medicaid test. Minnesota's Medical Assistance covers children ages 6-18 — PolicyEngine's OLDER_CHILD category — far above this household's $42,664 for three people (~160% FPL), so the child is Medicaid-eligible and thereby barred from CHIP. Every wrong model instead applied a standalone CHIP income ceiling, most of them quoting Minnesota's 275%/283% FPL child limits, which are the Medical Assistance and MinnesotaCare bands rather than CHIP bands. They read ""income below the child-coverage limit"" as qualifying when it is precisely what disqualifies: several stated the Medicaid fact outright and then used it backwards." diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv index 6fa3629e..4104af85 100644 --- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv +++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_case_reference_explanations.csv @@ -1809,7 +1809,7 @@ us,scenario_099,child2_medicaid_eligible,0.0,1,"PolicyEngine determined that chi us,scenario_020,head_medicaid_eligible,0.0,7,"PolicyEngine determined that the head of household is not eligible for Medicaid in Texas for 2026, resulting in `is_medicaid_eligible = False`. The person's `medicaid_category` evaluated to 9, indicating they qualify through none of the standard Medicaid eligibility pathways. While the person met the immigration status requirement (`is_medicaid_immigration_status_eligible = True`), having been in the US for 5 years, this alone was insufficient for eligibility. Additionally, the person did not qualify for California's FFYP program (`ca_ffyp_eligible = False`) or Illinois's HBI program (`il_hbi_eligible = False`), which might have provided alternative pathways. With a MAGI income level of 18.81 times the federal poverty line and no qualifying category, the household falls outside Texas's Medicaid eligibility criteria for 2026.", us,scenario_057,head_medicaid_eligible,1.0,1,"PolicyEngine determined that the head of household is eligible for Medicaid (is_medicaid_eligible = True) based on their status as an SSI recipient. Under Louisiana's Medicaid rules for 2026, individuals receiving Supplemental Security Income (SSI) are automatically enrolled in the SSI_RECIPIENT medicaid category, which carries automatic Medicaid eligibility regardless of income level. Although the household's Modified Adjusted Gross Income (MAGI) is measured at 0.45 times the Federal Poverty Level, this income threshold is not the binding constraint for SSI recipients—categorical eligibility through SSI receipt itself is sufficient to establish Medicaid eligibility. Therefore, PolicyEngine returned a value of 1.0 (True) for the head's Medicaid eligibility status.", us,scenario_123,spouse_medicaid_eligible,0.0,1,"PolicyEngine determined that the spouse is not eligible for Medicaid in Pennsylvania for 2026. The spouse, age 39, has a Modified Adjusted Gross Income (MAGI) of 5.31 times the Federal Poverty Level, which exceeds the income thresholds for all available Medicaid eligibility pathways in the state. With household income around $145,002, the spouse does not qualify through any of Pennsylvania's Medicaid categories—including those for parents, pregnant individuals, children, or disabled individuals—as their income substantially surpasses the relevant limits for each pathway. Consequently, the engine assigned an eligibility status of False with no qualifying category.", -us,scenario_031,head_medicaid_eligible,1.0,13,"PolicyEngine determined that the head of household is eligible for Medicaid in California as a senior-or-disabled individual. The person, age 67, qualifies through the optional senior-or-disabled category (a non-MAGI pathway) because they meet three conditions: they are aged or disabled (is_ssi_aged_blind_disabled = True), they satisfy the income test under this category (is_optional_senior_or_disabled_income_eligible = True), and they meet the asset limits (is_optional_senior_or_disabled_asset_eligible = True). Additionally, the person satisfies California's immigration status requirements, having been in the United States for 5 years. Under the senior-or-disabled category, countable income is calculated by deducting health insurance premiums, including Medicare Part B, from gross income, which results in the household falling within California's income limits for this eligibility group.", +us,scenario_031,head_medicaid_eligible,1.0,13,"PolicyEngine determined that the head of household is eligible for Medicaid in California through the optional senior-or-disabled category. At 67 the head is aged, which meets the category's first condition (is_ssi_aged_blind_disabled = True) without SSI receipt or a disability; the adult expansion group (ages 19 to 64) does not apply, and in the engine's Medicaid rules Medicare eligibility bars only that group. Income follows SSI rules: Social Security ($9,445.47, counted in full), the private pension ($13,608) and the IRA distribution ($800) total $23,853.47; the $1,165 of alimony paid is not subtracted. California's $230 monthly income disregard ($2,760 a year), which takes the place of SSI's $20 general income exclusion, leaves countable income of $21,093.47, within the $22,024.80 limit, 138% of the 2026 federal poverty guideline for one person (is_optional_senior_or_disabled_income_eligible = True). Countable resources are the $4,200 bank balance, under California's $130,000 limit for an individual (is_optional_senior_or_disabled_asset_eligible = True). The household lists no immigration status, so the engine's default status (citizen) applies and the immigration test passes.", us,scenario_071,head_medicaid_eligible,0.0,7,"PolicyEngine determined that this 76-year-old New York resident is not eligible for Medicaid (is_medicaid_eligible = False) because they do not qualify through any of the available eligibility pathways. While the person meets the immigration status requirement for Medicaid (is_medicaid_immigration_status_eligible = True, having been in the US for 5 years), they fail to qualify under the primary eligibility categories tracked by the engine. Specifically, the person does not qualify for California's FFYP program (ca_ffyp_eligible = False) or Illinois's HBI program (il_hbi_eligible = False), and their medicaid_category is assigned as NONE, indicating no qualifying pathway. With a MAGI income level of 2.28 times the Federal Poverty Level, the household's income and other circumstances do not align with any of New York's Medicaid eligibility categories for this household composition and age.", us,scenario_085,head_medicaid_eligible,0.0,7,"PolicyEngine determined that the head of household is not eligible for Medicaid in Pennsylvania in 2026, despite meeting the immigration status requirement. The person qualifies as immigration-eligible (having been in the US for 5 years), but this alone is insufficient for Medicaid eligibility. The engine evaluated two state-specific pathways—California's FFYP (Family Planning) program and Illinois's HBI (Health Benefits for Immigrants) program—both of which returned false, indicating the household does not qualify through these mechanisms. With a MAGI income of 3.69 times the federal poverty level and no qualifying Medicaid category identified, the person falls outside Pennsylvania's Medicaid eligibility pathways for 2026, resulting in an eligibility determination of False.", us,scenario_084,head_medicaid_eligible,0.0,7,"PolicyEngine determined that the head of household is not eligible for Medicaid in North Carolina in 2026, despite meeting the immigration status requirement. The person qualifies as immigration-eligible (having been in the US for 5 years), but this alone is insufficient for Medicaid eligibility. The engine assigned a medicaid_category value of 9, indicating no qualifying eligibility pathway, which directly resulted in is_medicaid_eligible being set to False. With a Modified Adjusted Gross Income (MAGI) of 1.56 times the Federal Poverty Level and no categorical eligibility through either the California FFYP or Illinois HBI programs, the household falls outside North Carolina's Medicaid coverage groups for this income and household composition.", diff --git a/paper/snapshot/20260501/manifest.json b/paper/snapshot/20260501/manifest.json index 97b18dde..620d046e 100644 --- a/paper/snapshot/20260501/manifest.json +++ b/paper/snapshot/20260501/manifest.json @@ -25,9 +25,9 @@ }, "files": { "us_adjudications.json": "42d6a90fe402b6dc8a37422999ea05cd0e481a5ead690ee4045f649d682270c6", - "us_audit_row_annotations.csv": "3b27bfa643a5725935e90bea39f2584dc507572418a8e1474e981b841078d3ad", - "us_case_notes.csv": "2bef75c043ec08338038276e018f300ed86fcb937c44ee26d0daadb58d7f2c08", - "us_case_reference_explanations.csv": "6cccc5586815f2ecc4f1b527e79e141f858d0e75cc30b3a867d107909cbb38c4" + "us_audit_row_annotations.csv": "ef393cc0604fe3ef57a1dc0eb10395405c818521ec172e29a00748b6e8657f0f", + "us_case_notes.csv": "fa26630bcdace072230beb23f4699e3f52f1ed6dba45cd8a89e745d4d4de00b5", + "us_case_reference_explanations.csv": "c5905eff49c28bf7fec436e49b5608f298eb8b3fb110cf35e214f4daacb0b275" }, "judge_provenance": { "by_judge": { diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/README.md b/reference_audit/2026-10-05-medicaid-031-annotations/README.md new file mode 100644 index 00000000..3745e2ea --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/README.md @@ -0,0 +1,91 @@ +# scenario_031 Medicaid annotations, October 5, 2026 + +This directory corrects the published audit text for one output: scenario_031's `head_medicaid_eligible` (California, single head age 67). The case note, the reference explanation and all 43 row annotations explained the reference (eligible) by subtracting a Medicare Part B premium from countable income. The reference system subtracts no premium. It applies California's $230 monthly income disregard instead. + +The rewrite changes text only. No reference, exclusion, failure class or score changes, and the frozen run directory is untouched. + +The finding came from the Medicare Part B audit (`../2026-10-05-medicare-part-b/`, PolicyEngine/policybench#193). + +## What the engine does + +These files hold for policyengine-us 2.15.17 as installed in the reference venv (`results/local/adds202609/triage/.venv-pe21517`). The reference system is that version plus `latest_final` (`../2026-09-28/fixes/`), on the household `Scenario.to_pe_household` builds. `scripts/engine_values.py` recomputes every figure below on that system and asserts each relation the new text states. + +- **Category.** `medicaid_category` is `SENIOR_OR_DISABLED`. + - `is_optional_senior_or_disabled_for_medicaid` requires three things: `is_ssi_aged_blind_disabled`, the income test and the asset test. + - `is_ssi_aged` is age 65 or older (`gov.ssa.ssi.eligibility.aged_threshold`), so age 67 meets the first condition. No SSI receipt or disability is needed. + - The adult expansion group does not apply. `is_adult_for_medicaid_nfc` requires age 19 to 64 (`categories.adult.age_range`) and no `is_medicare_eligible`. + - In `variables/gov/hhs/medicaid` that is the only formula that bars on Medicare eligibility. `medicaid_work_requirement_eligible` reads it as an exemption. `is_medicaid_eligible` itself is a category other than `NONE` plus immigration-status eligibility. + - No immigration status is listed, so the default, `CITIZEN`, applies. +- **Income.** `medicaid_optional_senior_or_disabled_countable_income` (`variables/gov/hhs/medicaid/income/`) adds three amounts: + - `ssi_marital_unearned_income`; + - income deemed from an ineligible parent; + - in-kind support and maintenance. + + The last two are 0 here, as is the income deemed from an ineligible spouse, which the formula adds after the disregard. SSI unearned income (`gov.ssa.ssi.income.sources.unearned`) includes `social_security`, `pension_income` and `retirement_distributions`. For this head that is $9,445.47 + $13,608 + $800 = $23,853.47. Social Security counts in full, and nothing is subtracted for the $1,165 of alimony paid. +- **Disregard.** The formula reads `gov.hhs.medicaid.eligibility.categories.senior_or_disabled.income.disregard.individual[state]`. For CA that is $230 a month. `_apply_medicaid_optional_senior_or_disabled_exclusions` passes it to `_apply_ssi_exclusions` as `general_exclusion`. That function uses SSI's $20 general exclusion only when no `general_exclusion` is given, so the $230 replaces the $20; it is not added to it. Countable income = $23,853.47 − 12 × $230 = $21,093.47. +- **Limit.** `medicaid_optional_senior_or_disabled_income_limit` is `income.limit.individual['CA']` (1.38) times the poverty guideline for one person: 1.38 × $15,960 (2026) = $22,024.80. `is_optional_senior_or_disabled_income_eligible` passes when income is at or below the limit. Countable income is $931.33 under it. +- **Assets.** `ssi_countable_resources` adds bank, stock and bond assets, $4,200 here. `assets.limit.individual['CA']` is $130,000 for 2026, and the test is assets below the limit. +- **Part B.** The engine treats the head as enrolled in Medicare and models a $2,434.80 Part B premium, but countable income does not read it. With `medicare_part_b_premium` at 0, or `takes_up_medicare_if_eligible` false, countable income is still $21,093.47 and the head is still eligible (`verification/engine_values.json`, readings `no_part_b` and `not_enrolled`). + +| | Annual | Monthly | +|---|---:|---:| +| SSI unearned income | $23,853.47 | $1,987.79 | +| less California's disregard | $2,760.00 | $230.00 | +| countable income | $21,093.47 | $1,757.79 | +| limit (138% of $15,960) | $22,024.80 | $1,835.40 | + +Gross income is $1,828.67 over the limit, and income less only SSI's $20 exclusion ($23,613.47) is still over it. A model that counted income under SSI rules but skipped California's disregard therefore fails the income test. + +The old text compared about $21,400 (gross less the Part B premium) with "about $21,600". That figure is 138% of the 2025 guideline ($15,650); the engine uses the 2026 guideline. + +## Where the old mechanism came from + +The judge's prompt for this case stated the wrong mechanism twice. + +- **The grounding line.** The grounding row for this case in `results/local/unified_audit/grounding.csv` (untracked; `GROUNDING_SHA256` in `scripts/finish_gpt61sol.py` pins it) describes the category as "a non-MAGI pathway whose income counting deducts health insurance premiums (including Medicare Part B) from countable income". No other case's grounding carries that phrase. +- **The reference explanation.** The old explanation said that countable income "is calculated by deducting health insurance premiums, including Medicare Part B". + +Both are in the prompt of the verdict behind the published text (`results/local/gpt61sol-stage/gpt61sol-v1/audit/cases/us__scenario_031__head_medicaid_eligible/prompt.md`, lines 25 and 29). That verdict was an isolated, tool-less Opus 5.5 run (`docs/gpt61sol/judge_provenance.json`), so the prompt was all it saw. This rewrite corrects the reference explanation. A future audit stage should also regenerate that grounding line; this directory does not touch the untracked file, because a driver pins its hash. + +## What changed + +`rewrites.json` holds 45 wording-only rewrites. Each replaces a field's whole old text with its whole new text: + +- the case note (`us_case_notes.csv`); +- the reference explanation (`us_case_reference_explanations.csv`); +- the 43 row annotations (`us_audit_row_annotations.csv`). + +`scripts/apply_rewrites.py` applies them and re-pins the three files' sha256 in `paper/snapshot/20260501/manifest.json` (`audit_annotation_artifacts.files`). It refuses a row that is missing or not unique, or a field that holds neither the old nor the new text. The CSVs round-trip byte for byte, so nothing else in them changes. With `--check` it writes nothing and confirms every rewrite is in force. `tests/test_annotation_rewrites.py` keeps the ledger in force, so a release that rebuilds these files from an older base cannot silently restore the old text. + +Every row stays a model error. `failure_source` (`llm_error`), `failure_subtype` and `reference_suspect` are unchanged; every model answered 0 against a reference of 1. The subtypes were re-checked against the corrected mechanism: + +- **29 rows, `taxable_income_or_deductions`.** These models miscounted income. 17 compared income with no exclusion or disregard with the limit, 2 applied only SSI's $20 exclusion, 2 cited an exclusion of unstated size, and 8 asserted without figures that income was too high. Skipping California's disregard is still an income-counting error. Corpus practice puts countable-income errors on Medicaid rows in this subtype (97 rows, these 29 included); one Medicaid row uses `state_local_rule`. +- **4 rows, `thresholds_rates`.** Wrong threshold: 100% of the guideline, or an SSI income limit. +- **3 rows, `asset_resource`.** SSI's $2,000 resource limit, or another asset error. +- **7 rows, `categorical_eligibility`.** These models missed that age alone gives the head a category. + +These counts equal the recorded subtype counts (`verification/row_review.json`, `tally`). + +## What did not change + +- **The frozen run.** The run directory `paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace/`, including `data.json.gz`, is untouched. The published payload (release `dashboard-data-20260930`) keeps the old text until a data release re-exports it. +- **Scores.** `scripts/payload_diff.py` rebuilds the payload from the frozen run with `export_country`, once with the annotations at `origin/main` and once with this tree's. The two differ in 132 paths, all in the scenario_031 `head_medicaid_eligible` cell, and only in its text fields: + - `annotation`, 43; + - `caseAnnotation`, 43; + - `referenceExplanation`, 46. + + No score, model statistic or other cell changes (`verification/payload_diff.json`). The base rebuild reproduces the frozen payload except claude-fable-5's usage fields, which the release drivers carry from the published payload (`CARRIED_USAGE`). + +## How the text was checked + +1. `scripts/engine_values.py` recomputed the engine values on the reference system and asserted each relation (`verification/engine_values.json`). +2. A Claude Code workflow drafted each row from the model's frozen response (`predictions.csv.gz`), against a fact sheet limited to what step 1 verified. A separate agent then checked every claim in each draft. It read the model's response itself, quoted the words each claim rests on, and corrected the draft. A final critic checked the rows against each other and against the case note and explanation. The record, with each row's evidence and the edits made after review, is `verification/row_review.json`. +3. An independent Subfleet review checked the result (`verification/independent_review.md`). + +## Related findings, not changed here + +- **scenario_114.** Both of its income tax outputs are proposed for exclusion (#191, #193). The engine check of its two case notes and their rows is posted on #193 for the release that adopts those records, rather than rewritten here: + - **Virginia note.** Group (4) blames models for leaving out the $2,000 of over-the-counter expenses or applying the 0.5%-of-AGI charitable floor. The reference itself does both. The gap is the $2,434.80 Part B premium the engine counts in the medical deduction. Eleven Virginia rows repeat the error. + - **Federal note.** Group (3) ("missing the Part B premium") is right about the engine but rests on that unlisted premium. +- **Over-the-counter expenses elsewhere.** Seven case notes and rows in scenarios 016, 075 and 121 count over-the-counter expenses in the itemized medical deduction. `itemized_medical_expenses` adds only `medical_expense_health_insurance_premiums` and `other_medical_expenses`. No stated conclusion changes. Split into a separate task. +- **Structured value and explanation trailer.** On this output, claude-opus-4.8 and claude-sonnet-4.6 submitted 1 in the tool call, but their explanations end "value = 0", and the frozen prediction is 0. `eval_no_tools.py` now treats a valid structured value as canonical. Whether the frozen rows should be re-parsed is a scoring question, split into a separate task. The row text here describes the explanations as written. diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/rewrites.json b/reference_audit/2026-10-05-medicaid-031-annotations/rewrites.json new file mode 100644 index 00000000..e3b574d9 --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/rewrites.json @@ -0,0 +1,453 @@ +{ + "note": "Wording-only rewrites of the frozen audit annotations for scenario_031 head_medicaid_eligible (scripts/apply_rewrites.py applies them; tests/test_annotation_rewrites.py keeps them in force). Each replaces a field's whole old text with its whole new text. None changes a failure_source, failure_subtype, reference_suspect flag, exclusion, reference or score.", + "rewrites": [ + { + "file": "us_case_notes.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "field": "case_annotation", + "old": "At 67, the head qualifies as aged, so the engine tests Medi-Cal through the optional senior-or-disabled pathway (California's Aged & Disabled FPL program). This is a non-MAGI pathway with a 138% FPL income limit. Countable income is gross income minus health insurance premiums, including the Medicare Part B standard premium (about $202.90/month, or about $2,435/year in 2026). That brings the $23,853 gross down to about $21,400, below 138% FPL (about $21,600). The $4,200 in bank assets easily passes California's asset test. Almost every model compared gross or near-gross income with the limit and never subtracted the Part B premium. Others used the wrong threshold (100% FPL or the SSI rate), applied the $2,000 SSI resource limit, or denied that an aged person has a Medicaid category.", + "new": "At 67 the head is aged (65 or older), which alone meets the category condition of the engine's optional senior-or-disabled pathway; it needs no disability or SSI receipt. The adult expansion group (ages 19 to 64, no Medicare eligibility) does not apply, but in the engine's Medicaid rules Medicare eligibility bars only that group. The pathway counts income under SSI rules: Social Security ($9,445.47, in full), the $13,608 pension and the $800 IRA distribution give $23,853.47; the $1,165 of alimony paid is not subtracted. In place of SSI's $20 monthly general income exclusion, the engine applies California's $230 monthly disregard ($2,760 a year), leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline of $15,960). The $4,200 in bank assets is far below California's $130,000 asset limit for the pathway. The engine also models a $2,434.80 Medicare Part B premium for the head, but countable income does not subtract it: with the premium at zero, or the head not enrolled in Medicare, countable income is still $21,093.47. Most models (29 of 43) miscounted income: most of those compared about $23,853 with the limit, a few applied only SSI's $20 exclusion or an exclusion of unstated size, and several asserted without figures that income was too high. The rest used the wrong threshold (100% of the poverty guideline or an SSI income limit), applied SSI's $2,000 resource limit or otherwise judged the $4,200 too much, or denied the head a category, by treating Medicare eligibility or the lack of SSI receipt or a disability as disqualifying, or by testing only the adult expansion group.", + "reason": "The old note said countable income subtracts health insurance premiums, including the Medicare Part B premium, and compared about $21,400 with a 138% FPL limit of about $21,600 (2025 guideline). The engine applies California's $230 monthly disregard and the 2026 guideline: $21,093.47 against $22,024.80." + }, + { + "file": "us_case_reference_explanations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "field": "explanation", + "old": "PolicyEngine determined that the head of household is eligible for Medicaid in California as a senior-or-disabled individual. The person, age 67, qualifies through the optional senior-or-disabled category (a non-MAGI pathway) because they meet three conditions: they are aged or disabled (is_ssi_aged_blind_disabled = True), they satisfy the income test under this category (is_optional_senior_or_disabled_income_eligible = True), and they meet the asset limits (is_optional_senior_or_disabled_asset_eligible = True). Additionally, the person satisfies California's immigration status requirements, having been in the United States for 5 years. Under the senior-or-disabled category, countable income is calculated by deducting health insurance premiums, including Medicare Part B, from gross income, which results in the household falling within California's income limits for this eligibility group.", + "new": "PolicyEngine determined that the head of household is eligible for Medicaid in California through the optional senior-or-disabled category. At 67 the head is aged, which meets the category's first condition (is_ssi_aged_blind_disabled = True) without SSI receipt or a disability; the adult expansion group (ages 19 to 64) does not apply, and in the engine's Medicaid rules Medicare eligibility bars only that group. Income follows SSI rules: Social Security ($9,445.47, counted in full), the private pension ($13,608) and the IRA distribution ($800) total $23,853.47; the $1,165 of alimony paid is not subtracted. California's $230 monthly income disregard ($2,760 a year), which takes the place of SSI's $20 general income exclusion, leaves countable income of $21,093.47, within the $22,024.80 limit, 138% of the 2026 federal poverty guideline for one person (is_optional_senior_or_disabled_income_eligible = True). Countable resources are the $4,200 bank balance, under California's $130,000 limit for an individual (is_optional_senior_or_disabled_asset_eligible = True). The household lists no immigration status, so the engine's default status (citizen) applies and the immigration test passes.", + "reason": "The old explanation said countable income deducts health insurance premiums, including Medicare Part B, and that the head had been in the United States for 5 years. The engine deducts no premium; it applies California's $230 monthly disregard. No immigration status is listed, so the engine's default status (citizen) applies." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-fable-5", + "field": "annotation", + "old": "It correctly picked the aged pathway and its 138% FPL limit. But it built countable income by subtracting only alimony ($22,688) and never subtracted the Medicare Part B premium (about $2,435/year). With that deduction, countable income is about $21,400, which is under the limit.", + "new": "It took the aged pathway and a 138% limit of about $1,800 a month, subtracted the $1,165 of alimony to put income at about $1,890 a month, and assumed no specific disregard, saying even SSI-style disregards would leave it over. The engine does not subtract alimony; it takes $1,987.79 a month less California's $230 disregard, which replaces SSI's $20 exclusion, for $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-fable-5.1", + "field": "annotation", + "old": "It applied only the $240 general income exclusion ($23,613) against the 138% FPL aged/disabled limit. It left out the senior-or-disabled deduction for health insurance premiums, including Medicare Part B (about $2,435/year), which puts countable income below the roughly $21,600 limit.", + "new": "It used the aged pathway but applied only SSI's $240 annual general exclusion, putting countable income at $23,613 against a 138% limit of about $21,600. California's $230 monthly disregard takes the place of that $20 exclusion, so the engine counts $23,853.47 less $2,760, or $21,093.47, which is under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-haiku-4.5", + "field": "annotation", + "old": "It compared gross income of $23,853 with a 138% FPL figure understated at $18,564 and never deducted the Medicare Part B premium from countable income. With Part B subtracted, countable income is about $21,400, under 138% FPL (about $21,600). It also doubted the $4,200 in assets, although that amount easily passes California's senior-or-disabled asset test.", + "new": "It compared gross income of $23,853, and that amount less the $1,165 of alimony, against a 138% limit it understated at about $18,564, and doubted the $4,200 in assets. The engine subtracts no alimony: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline). The $4,200 is far under California's $130,000 asset limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-opus-4.8", + "field": "annotation", + "old": "It changed its answer to a 100% FPL limit (about $15,650). California's senior-or-disabled pathway uses 138% FPL. It also counted the full $23,853 without deducting the Medicare Part B premium, which brings countable income to about $21,400, within the limit.", + "new": "It first called gross income of $23,853 well below the threshold, then switched to a MAGI-based 100% limit of about $15,650 (the 2025 guideline), and its explanation concluded the head was ineligible. The engine's senior pathway uses SSI income rules and 138% of the 2026 guideline, $22,024.80; it counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-opus-5", + "field": "annotation", + "old": "It tested gross income of $23,853 against a 100–138% FPL range and never subtracted health insurance premiums. The non-MAGI senior-or-disabled count deducts the Medicare Part B premium (about $2,435/year), which puts income under the 138% FPL limit.", + "new": "It tested gross income of $23,853 against an aged limit it gave as 100% to 138% of the poverty line, about $15,650 to $21,600 (2025-guideline figures), and found it over. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-opus-5.5", + "field": "annotation", + "old": "It used the correct 138% FPL aged limit but compared it with gross income of $23,853. Under this pathway, countable income is net of the Medicare Part B premium (about $2,435/year), leaving about $21,400, which is below the limit.", + "new": "It used the 138% aged limit, at about $21,600 (the 2025-guideline figure), but compared it with gross income of $23,853. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under its $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-sonnet-4.6", + "field": "annotation", + "old": "It switched between the ACA expansion test, which does not cover someone aged 67, and a 100% FPL aged limit. It settled on gross $23,853 against an understated 138% FPL figure of $20,783. It never used the senior-or-disabled pathway's deduction of the Medicare Part B premium, which brings countable income under the real 138% FPL limit.", + "new": "It moved between the adult expansion test, which excludes someone aged 67, and a 100% aged limit of $15,060; its explanation settled on the expansion test, with gross income of $23,853 against $20,783, which is 138% of its understated $15,060 guideline. The engine's senior pathway counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under $22,024.80 (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-sonnet-5", + "field": "annotation", + "old": "It first stated the correct Aged & Disabled FPL limit (about $21,597), which its own $14,408 figure would pass. Then it switched to the SSI federal benefit rate (about $13,596) as the binding limit. It also treated Social Security as MAGI-excluded, although the non-MAGI count includes Social Security in full and instead deducts the Medicare Part B premium.", + "new": "It excluded most Social Security to get $14,408, said 138% applied only to expansion adults under 65, cited a 138% aged limit of about $21,597 that $14,408 would pass, then used an SSI limit of about $13,596 as binding. The engine counts Social Security in full and uses 138% of the 2026 guideline, $22,024.80: $23,853.47 less California's $230 monthly disregard ($2,760 a year) gives $21,093.47, under the limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "claude-sonnet-5.5", + "field": "annotation", + "old": "It said the head is in no categorical group, but at 67 the head is aged, which is exactly the senior-or-disabled category. It also used unadjusted income of about $23.9k and skipped the Medicare Part B premium deduction, which brings countable income below 138% FPL.", + "new": "It said the head was in no categorical group and that countable income of about $23.9k, including Social Security and with no disregard, exceeded 138% FPL. At 67 the head is aged, which alone meets the category condition of the engine's senior-or-disabled pathway, and the engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "deepseek-v4-flash-0731", + "field": "annotation", + "old": "It compared gross income of $23,853 with 138% FPL without deducting health insurance premiums. The senior-or-disabled pathway subtracts the Medicare Part B premium (about $2,435/year), leaving countable income of about $21,400, under the limit.", + "new": "It compared gross income of about $23,853 with 138% FPL and concluded the head was over the limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "deepseek-v4-pro", + "field": "annotation", + "old": "It used gross monthly income of $1,987.75 against an understated 138% FPL of $1,732/month. It never deducted the Medicare Part B premium (about $203/month), which brings countable income to about $1,785/month, under the correct 138% FPL of about $1,800/month.", + "new": "It tested gross monthly income of $1,987.75 against an understated 138% FPL limit of $1,732. The engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $1,757.79, under the actual limit of $1,835.40 a month (138% of the 2026 guideline). Both errors mattered: income after the disregard still tops the model's $1,732 figure.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "deepseek-v4-pro-0813", + "field": "annotation", + "old": "It treated the full $23,853 as countable income and required SSI receipt for a categorical route. The senior-or-disabled pathway does not require SSI and deducts the Medicare Part B premium, leaving countable income below 138% FPL.", + "new": "It treated the full $23,853 as countable income and found it over California's FPL-based limit for a senior, without stating the limit. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "deepseek-v4.1-flash", + "field": "annotation", + "old": "It compared gross income of $23,853 with its own 138% FPL figure of $22,218. Deducting the Medicare Part B premium (about $2,435/year), as the senior-or-disabled income count requires, leaves about $21,400, below even the model's own threshold.", + "new": "It compared gross income of $23,853 with its own 138% FPL figure of $22,218, slightly above the engine's $22,024.80 limit (138% of the 2026 guideline). The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under either figure.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3-flash-preview", + "field": "annotation", + "old": "It picked the correct 138% FPL Aged & Disabled limit (about $22,080 by its estimate) but tested gross income of $23,853. The pathway deducts Medicare Part B premiums from countable income, which brings it to about $21,400, under the limit.", + "new": "It used a 138% FPL limit of about $22,080, close to the engine's $22,024.80 (138% of the 2026 guideline), but tested gross income of $23,853 against it. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves countable income of $21,093.47, under the limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3.1-flash-lite-preview", + "field": "annotation", + "old": "It gave no derivation, and its 'No' is consistent with comparing gross income of $23,853 with an FPL limit. The senior-or-disabled pathway subtracts the Medicare Part B premium, which leaves countable income of about $21,400, below 138% FPL, so the head is eligible.", + "new": "It gave no figures or pathway, saying only that household income exceeded the Medicaid limit \"for a single adult.\" At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3.1-pro-preview", + "field": "annotation", + "old": "It tested gross income of $23,853 against the 138% FPL Aged & Disabled limit without deducting health insurance premiums. Subtracting the Medicare Part B premium (about $2,435/year) puts countable income under the limit.", + "new": "It compared gross income of $23,853 with 138% FPL, without stating the dollar limit, and found it over. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3.5-flash", + "field": "annotation", + "old": "It correctly identified the non-MAGI 138% FPL limit but measured gross monthly income against it. The non-MAGI senior-or-disabled count deducts the Medicare Part B premium, which brings income below the limit.", + "new": "It correctly picked the 138% FPL limit for what it called \"free non-MAGI Medi-Cal\" but gave no figures, saying only that the head's monthly income was over it; in the same response it put gross income at $23,853 for its federal tax and SNAP answers. The engine subtracts California's $230 monthly disregard from $1,987.79 a month, in place of SSI's $20 exclusion. That leaves $1,757.79 a month, under the $1,835.40 monthly limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3.5-flash-lite", + "field": "annotation", + "old": "It gave no derivation. The correct path is the aged (senior-or-disabled) category: $23,853 gross minus the Medicare Part B premium gives about $21,400, under 138% FPL, with assets passing. The 'No' reflects skipping that premium deduction.", + "new": "It gave no figures, threshold or pathway, citing only the head's age and income. At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) is $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3.6-flash", + "field": "annotation", + "old": "It compared the full $23,853 with the aged Medi-Cal threshold without subtracting the Medicare Part B premium. The senior-or-disabled pathway deducts that premium, leaving countable income below 138% FPL.", + "new": "It treated the full $23,853 as countable and said it exceeded an unstated threshold for people 65 or older. Gross income is $1,828.67 over the limit, but the engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3.7-flash", + "field": "annotation", + "old": "It correctly identified the non-MAGI elderly pathway but did not deduct health insurance premiums, including Medicare Part B, from countable income. That deduction brings income from $23,853 to about $21,400, under the 138% FPL limit.", + "new": "It placed the head in what it called the \"non-MAGI\" elderly pathway but gave no income figure or limit, and it judged income too high, which holds against the engine's limit only if California's $230 disregard is skipped or replaced by a smaller one, such as SSI's $20. In the engine's senior-or-disabled pathway, $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gemini-3.8-flash", + "field": "annotation", + "old": "It said total income 'well' exceeds 138% FPL, but gross income is only about 150% FPL. After the senior-or-disabled pathway deducts the Medicare Part B premium, countable income falls below 138% FPL.", + "new": "It tested total income against a 138% FPL limit with no disregard and called it \"well\" over, but $23,853.47 is about 149% of the 2026 guideline, only $1,828.67 over the $22,024.80 limit. The engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "glm-5.2", + "field": "annotation", + "old": "It assumed Medicare eligibility rules out Medicaid. Medicare-eligible seniors qualify for Medicaid (dual eligibility) through the aged senior-or-disabled pathway, which this head meets on income after the Part B deduction and on assets.", + "new": "It assumed that Medicare eligibility at 65 or older bars Medicaid and ran no income test. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group, and age 67 alone meets the aged test for the senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "glm-5.3", + "field": "annotation", + "old": "It tested the head against the ACA adult expansion limit, which does not apply at age 67, and overstated income as about 199% FPL; $23,853 is about 150% FPL. The applicable pathway is senior-or-disabled, where countable income after the Medicare Part B premium deduction is under 138% FPL.", + "new": "It applied the adult expansion limit, which excludes anyone over 64, comparing gross income of about $1,988 a month (about 149% of the 2026 guideline, not the 199% it stated) with an understated limit of about $1,600. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $1,987.79 less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-5.4-nano", + "field": "annotation", + "old": "It looked for a disability flag and missed that age 67 alone meets the aged criterion for the senior-or-disabled pathway. Under that pathway, income net of the Medicare Part B premium is below 138% FPL and the $4,200 in assets passes the asset test.", + "new": "It denied eligibility because no disability or low-income flag was listed and income was \"substantial\", without comparing any income figure with a limit. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-5.5", + "field": "annotation", + "old": "It applied a spend-down (medically needy) standard and counted only the $50 over-the-counter expense as a medical deduction. The applicable pathway is the 138% FPL senior-or-disabled program, which deducts the Medicare Part B premium (about $2,435/year) and leaves countable income under the limit.", + "new": "It correctly ruled out the adult expansion group at age 67 but tested the full $23,853 against an unstated aged income or \"spend-down\" standard, with only minimal medical expenses to offset it. The engine's senior-or-disabled pathway instead subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-5.6-luna", + "field": "annotation", + "old": "It gave no computation, and its 'above threshold' conclusion matches testing gross income. The senior-or-disabled pathway subtracts the Medicare Part B premium from $23,853, leaving about $21,400, below 138% FPL.", + "new": "It gave no figures, pathway or threshold and only asserted that countable income was above the applicable California limit, which holds against the engine's limit only if the state's $230 monthly disregard ($2,760 a year) is skipped or replaced by a smaller one, such as SSI's $20. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $23,853.47 less that disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-5.6-sol", + "field": "annotation", + "old": "It correctly picked the aged pathway but compared unadjusted retirement income with the limit. That pathway deducts health insurance premiums, including Medicare Part B, which brings countable income under 138% FPL.", + "new": "It identified the aged pathway but concluded, without showing figures, that retirement income exceeded California's limit. That holds against the engine's limit only if California's $230 monthly disregard is skipped or replaced by a smaller one, such as SSI's $20: gross income of $23,853.47 is $1,828.67 over the limit. The engine's senior-or-disabled pathway subtracts that disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-5.6-terra", + "field": "annotation", + "old": "It used an SSI-linked income level as the aged Medicaid test. California's optional senior-or-disabled pathway uses 138% FPL, and after the Medicare Part B premium deduction the head's countable income is under that limit.", + "new": "It placed the head in the aged pathway but tested income against an \"SSI-linked income level,\" giving no figures. The engine's limit for this pathway in California is $22,024.80 (138% of the 2026 poverty guideline), and countable income is $21,093.47: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-6-astra", + "field": "annotation", + "old": "It applied only the $20/month general income exclusion and left out the senior-or-disabled deduction for health insurance premiums. Subtracting the Medicare Part B premium (about $2,435/year) brings countable income below 138% FPL.", + "new": "It correctly ruled out the adult expansion group at 67 but said retirement income after the general income exclusion exceeded California's aged limit, giving no figures. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-6-sol", + "field": "annotation", + "old": "It compared gross retirement income with the aged threshold without deducting the Medicare Part B premium. The senior-or-disabled income count requires that deduction, which leaves the head at about $21,400, under 138% FPL.", + "new": "It said the head's retirement income exceeded California's aged Medicaid threshold, naming no exclusion and giving no figures. Gross income of $23,853.47 is over the $22,024.80 limit (138% of the 2026 poverty guideline), but the engine first subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving countable income of $21,093.47, under the limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "gpt-6.1-sol", + "field": "annotation", + "old": "It deducted only the general income exclusion and missed the health insurance premium deduction, including Medicare Part B, that the senior-or-disabled pathway applies. With that deduction, countable income falls below 138% FPL.", + "new": "It said retirement income exceeded California's aged limit after \"the applicable general income exclusion,\" without stating its amount, the income or the limit. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "grok-4.3", + "field": "annotation", + "old": "It claimed PolicyEngine excludes Medicare-eligible (dual-eligible) people from Medicaid. The aged senior-or-disabled pathway is built for Medicare-age individuals, and this head meets its income test (after the Part B deduction) and its asset test.", + "new": "It ruled the head out as Medicare-eligible, claiming the engine excludes \"full dual eligibles\" from Medicaid. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group: at 67 the head counts as aged for the senior-or-disabled pathway, countable income of $21,093.47 ($23,853.47 less California's $230 monthly disregard, $2,760 a year) is under the $22,024.80 limit (138% of the 2026 poverty guideline), and $4,200 in assets is under the $130,000 limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "grok-4.5", + "field": "annotation", + "old": "It treated $4,200 in bank assets as over the aged resource limit, but $4,200 easily passes California's senior-or-disabled asset test ($130,000 for an individual in 2026). It also tested gross income of $23,853 without deducting the Medicare Part B premium, which brings countable income under 138% FPL.", + "new": "It said bank assets of $4,200 and countable income of about $23,853 both exceeded California's aged limits, counting gross income in full. The pathway's asset limit is $130,000, so $4,200 passes, and the engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "grok-4.6", + "field": "annotation", + "old": "It mixed MAGI and ABD counting and compared the full $23,853 with the Aged & Disabled FPL limit. The non-MAGI count deducts the Medicare Part B premium (about $2,435/year), leaving about $21,400, under 138% FPL.", + "new": "It counted the full $23,853 as \"MAGI/ABD countable income\" and said it exceeded California's \"Aged & Disabled FPL\" limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "grok-4.7", + "field": "annotation", + "old": "It applied only the $20 monthly unearned-income disregard before testing against the 138% FPL Aged & Disabled limit. It missed the deduction of health insurance premiums, including Medicare Part B, which brings countable income below the limit.", + "new": "It applied only the $20 monthly unearned-income disregard before testing against California's 138% FPL limit, which leaves about $23,613, still over the $22,024.80 limit. The engine uses California's $230 monthly disregard in place of that $20 exclusion, so countable income is $21,093.47 ($23,853.47 less $2,760), under the limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "grok-build-0.1", + "field": "annotation", + "old": "It applied the $2,000 SSI resource limit. California's senior-or-disabled Medi-Cal asset limit is $130,000 for an individual in 2026, so $4,200 passes, and income after the Medicare Part B deduction is under 138% FPL.", + "new": "It applied a $2,000 asset limit to aged Medicaid and denied eligibility because the $4,200 in bank assets exceeded it, but $2,000 is SSI's resource limit, not this pathway's test. The engine's California senior-or-disabled pathway allows assets below $130,000, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "inkling", + "field": "annotation", + "old": "It used a 100% FPL limit (about $15,650) for aged Medi-Cal, but California's senior-or-disabled pathway uses 138% FPL. It also counted gross income without deducting the Medicare Part B premium, which brings countable income to about $21,400, under the limit.", + "new": "It applied a 100% FPL limit of about $15,650, the 2025 guideline, and compared gross income of about $23,853 against it. The engine uses 138% of the 2026 guideline, $22,024.80, and subtracts California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) from $23,853.47, leaving countable income of $21,093.47, under the limit.", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "kimi-k2.6", + "field": "annotation", + "old": "It said the head fits no non-MAGI aged or disabled pathway because they are not disabled or on SSI. At 67 the head is aged, which qualifies them for the senior-or-disabled category without SSI receipt, and they pass its income test (after the Part B deduction) and its asset test.", + "new": "It correctly placed the head outside the adult expansion group but ruled out the aged and disabled pathways because the head was not disabled, blind or an SSI recipient. The engine's senior-or-disabled pathway needs no disability, blindness or SSI receipt; age 67 alone meets its aged criterion. Countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "kimi-k3", + "field": "annotation", + "old": "It correctly excluded the adult expansion group but compared unadjusted retirement income with the aged limit. The senior-or-disabled pathway deducts the Medicare Part B premium, which leaves countable income under 138% FPL.", + "new": "It correctly placed the head outside the adult expansion group but said the listed retirement income exceeded the California aged income limit, without naming a limit or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "minimax-m3", + "field": "annotation", + "old": "It guessed that roughly $24k of income exceeds the senior limit without using any threshold or deduction. The correct calculation subtracts the Medicare Part B premium (about $2,435) from $23,853, which gives about $21,400, under 138% FPL.", + "new": "It guessed that about $24,000 of income might exceed an unstated senior limit and answered 0 without naming a threshold or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving countable income of $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "ox-alpha", + "field": "annotation", + "old": "It applied the SSI $2,000 resource limit to the $4,200 in bank assets. California's senior-or-disabled asset test ($130,000 individual limit in 2026) is passed. It also compared gross income with the income standard without deducting the Medicare Part B premium, which brings income below 138% FPL.", + "new": "It correctly treated the head as aged but applied SSI's $2,000 resource limit to the $4,200 in bank assets and said gross income of about $24,000 exceeded the aged income standard. The engine's California pathway allows assets below $130,000 and subtracts a $230 monthly disregard ($2,760 a year) from the $23,853.47 of income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "qwen-3.7-max", + "field": "annotation", + "old": "It compared gross income of $23,853 with 'typical' ABD limits and treated missing SSI or disability status as disqualifying. The aged senior-or-disabled pathway requires neither, and it deducts the Medicare Part B premium, which leaves countable income under 138% FPL.", + "new": "It counted $23,853 of Social Security, pension and IRA income in full, said it exceeded \"typical aged/blind/disabled Medicaid limits\" without naming one, and cited the missing disability or SSI status. The engine's senior-or-disabled pathway needs neither, since age 67 alone meets its aged criterion, and it subtracts California's $230 monthly disregard ($2,760 a year) from $23,853.47, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + }, + { + "file": "us_audit_row_annotations.csv", + "scenario_id": "scenario_031", + "variable": "head_medicaid_eligible", + "model": "qwen3.8-max", + "field": "annotation", + "old": "It concluded the 67-year-old is in no eligibility category. Age 65+ is itself the aged criterion for the senior-or-disabled pathway, where income net of the Medicare Part B premium is below 138% FPL and assets pass.", + "new": "It said the 67-year-old was not disabled, blind or in any other listed category, and that income was above the senior pathway's limit without naming a figure. Age 65 or older alone meets the aged criterion of the engine's senior-or-disabled pathway, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "reason": "The old text explained the reference with a Medicare Part B premium subtracted from countable income; the engine subtracts no premium. It counts $23,853.47 of SSI unearned income and applies California's $230 monthly disregard in place of SSI's $20 exclusion, so countable income is $21,093.47 against a $22,024.80 limit (138% of the 2026 guideline). The row stays a model error." + } + ] +} diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/scripts/apply_rewrites.py b/reference_audit/2026-10-05-medicaid-031-annotations/scripts/apply_rewrites.py new file mode 100644 index 00000000..a0f58f09 --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/scripts/apply_rewrites.py @@ -0,0 +1,152 @@ +"""Apply rewrites.json to the frozen audit annotations and re-pin their hashes. + +Each rewrite names one annotation file, one row (scenario_id and variable, plus model for +row annotations), one text field, the field's whole old text and its whole new text. The +script refuses a rewrite whose row is missing or not unique, or whose field holds neither +the old nor the new text, and it changes nothing else: no other field, no other row, no +row order and no quoting. It then writes the new sha256 of each annotation file into +paper/snapshot/20260501/manifest.json (``audit_annotation_artifacts.files``). + +``--check`` writes nothing and fails unless every rewrite is in force and the manifest +pins the files as they are. + + python reference_audit/2026-10-05-medicaid-031-annotations/scripts/apply_rewrites.py + python reference_audit/2026-10-05-medicaid-031-annotations/scripts/apply_rewrites.py --check +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import io +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +HERE = Path(__file__).resolve().parents[1] +REWRITES = HERE / "rewrites.json" +MANIFEST = ROOT / "paper/snapshot/20260501/manifest.json" + +KEYS = { + "us_audit_row_annotations.csv": ("scenario_id", "variable", "model"), + "us_case_notes.csv": ("scenario_id", "variable"), + "us_case_reference_explanations.csv": ("scenario_id", "variable"), +} +FIELDS = { + "us_audit_row_annotations.csv": "annotation", + "us_case_notes.csv": "case_annotation", + "us_case_reference_explanations.csv": "explanation", +} + + +def _read(path: Path) -> tuple[list[str], list[list[str]], bytes]: + raw = path.read_bytes() + rows = list(csv.reader(io.StringIO(raw.decode("utf-8"), newline=""))) + return rows[0], rows[1:], raw + + +def _write(header: list[str], rows: list[list[str]]) -> bytes: + out = io.StringIO(newline="") + csv.writer(out, lineterminator="\n").writerows([header, *rows]) + return out.getvalue().encode("utf-8") + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + + payload = json.loads(REWRITES.read_text()) + manifest = json.loads(MANIFEST.read_text()) + block = manifest["audit_annotation_artifacts"] + annotations = ROOT / block["path"] + problems: list[str] = [] + changed_files: dict[str, bytes] = {} + + by_file: dict[str, list[dict]] = {} + for item in payload["rewrites"]: + by_file.setdefault(item["file"], []).append(item) + + for name, items in sorted(by_file.items()): + if name not in KEYS: + problems.append(f"{name}: not a rewritable annotation file") + continue + header, rows, raw = _read(annotations / name) + # The round trip must reproduce the file, so a write changes only the + # rewritten fields. + if _write(header, rows) != raw: + problems.append(f"{name}: does not round-trip byte for byte") + continue + columns = {column: i for i, column in enumerate(header)} + key_columns = [columns[c] for c in KEYS[name]] + field = columns[FIELDS[name]] + index: dict[tuple, list[int]] = {} + for i, row in enumerate(rows): + index.setdefault(tuple(row[c] for c in key_columns), []).append(i) + seen: set[tuple] = set() + for item in items: + if item["field"] != FIELDS[name]: + problems.append(f"{name}: {item['field']!r} is not the text field") + continue + key = tuple(item[c] for c in KEYS[name]) + if key in seen: + problems.append(f"{name}: two rewrites of {key}") + continue + seen.add(key) + hits = index.get(key, []) + if len(hits) != 1: + problems.append(f"{name}: {key} matches {len(hits)} rows") + continue + current = rows[hits[0]][field] + if current == item["new"]: + continue + if current != item["old"]: + problems.append(f"{name}: {key} holds neither the old nor the new text") + continue + if args.check: + problems.append(f"{name}: {key} still holds the old text") + continue + rows[hits[0]][field] = item["new"] + data = _write(header, rows) + if data != raw: + changed_files[name] = data + + if problems: + print("\n".join(problems), file=sys.stderr) + return 1 + + if not args.check: + for name, data in changed_files.items(): + (annotations / name).write_bytes(data) + print(f"rewrote {name}") + + stale = [] + for name in sorted(KEYS): + digest = _sha256((annotations / name).read_bytes()) + if block["files"][name] != digest: + stale.append(name) + block["files"][name] = digest + if stale and args.check: + print(f"manifest pins stale hashes for {stale}", file=sys.stderr) + return 1 + if stale: + text = json.dumps(manifest, indent=2, ensure_ascii=False) + "\n" + original = MANIFEST.read_text() + # Keep the manifest's formatting: it must round-trip before we rewrite it. + if json.dumps(json.loads(original), indent=2, ensure_ascii=False) + "\n" != original: + print("manifest.json does not round-trip; refusing to rewrite it", file=sys.stderr) + return 1 + MANIFEST.write_text(text) + print(f"re-pinned {stale} in {MANIFEST.relative_to(ROOT)}") + print(f"{len(payload['rewrites'])} rewrites in force") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/scripts/engine_values.py b/reference_audit/2026-10-05-medicaid-031-annotations/scripts/engine_values.py new file mode 100644 index 00000000..de54e2ba --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/scripts/engine_values.py @@ -0,0 +1,294 @@ +"""Recompute how the reference system decides scenario_031's head Medicaid eligibility. + +The reference system is the one that built the published references: policyengine-us +2.15.17 plus ``latest_final`` (reference_audit/2026-09-28/fixes), on the household +``Scenario.to_pe_household`` builds. The script records the 2026 parameters and the +intermediate variables of the optional senior-or-disabled pathway, under the engine's +modeled Medicare Part B premium and under two readings without it: + +``no_part_b`` + The head's ``medicare_part_b_premium`` is 0. +``not_enrolled`` + The head's ``takes_up_medicare_if_eligible`` is False, so the head is not enrolled. + +It asserts what the rewritten annotations say: countable income is SSI unearned income +less 12 months of California's $230 disregard, which replaces SSI's $20 general income +exclusion; it is the same under every reading; it is at or below 138% of the 2026 +poverty guideline; gross income and income less only the $20 exclusion are above that +limit; countable resources are below California's asset limit; and the reference equals +the published value. + +Run from a policybench checkout with the policyengine-us 2.15.17 venv: + + PYTHONDONTWRITEBYTECODE=1 OPENBLAS_NUM_THREADS=1 PYTHONPATH= \\ + /.venv-pe21517/bin/python \\ + reference_audit/2026-10-05-medicaid-031-annotations/scripts/engine_values.py +""" + +from __future__ import annotations + +import copy +import importlib.util +import json +import shutil +import sys +import tempfile +from importlib.metadata import version +from pathlib import Path + +import numpy as np +import pandas as pd + +ROOT = Path(__file__).resolve().parents[3] +HERE = Path(__file__).resolve().parents[1] +RUN = ( + ROOT + / "paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace" +) +AUDIT_0928 = ROOT / "reference_audit/2026-09-28" +AUDIT_0922 = ROOT / "reference_audit/2026-09-22" +SCENARIO = "scenario_031" +OUTPUT = "head_medicaid_eligible" +YEAR = 2026 +PERIOD = str(YEAR) +STATE = "CA" +CENT = 0.005 + +READINGS = { + "modeled": {}, + "no_part_b": {"medicare_part_b_premium": 0.0}, + "not_enrolled": {"takes_up_medicare_if_eligible": False}, +} + +PERSON_VARIABLES = ( + "age", + "is_ssi_aged", + "is_ssi_disabled", + "is_blind", + "is_ssi_aged_blind_disabled", + "is_ssi_recipient_for_medicaid", + "is_medicare_eligible", + "medicare_enrolled", + "medicare_part_b_premium", + "social_security", + "pension_income", + "taxable_ira_distributions", + "alimony_expense", + "ssi_unearned_income", + "ssi_marital_unearned_income", + "ssi_marital_earned_income", + "ssi_in_kind_support_and_maintenance", + "medicaid_optional_senior_or_disabled_unearned_income_deemed_from_ineligible_parent", + "medicaid_optional_senior_or_disabled_income_deemed_from_ineligible_spouse", + "ssi_couple_computation_applies", + "medicaid_optional_senior_or_disabled_countable_income", + "medicaid_optional_senior_or_disabled_income_limit", + "is_optional_senior_or_disabled_income_eligible", + "ssi_countable_resources", + "is_optional_senior_or_disabled_asset_eligible", + "is_optional_senior_or_disabled_for_medicaid", + "is_adult_for_medicaid_nfc", + "is_adult_for_medicaid", + "immigration_status", + "is_medicaid_immigration_status_eligible", + "medicaid_category", + "is_medicaid_eligible", +) + + +def _system(): + from policyengine_us import CountryTaxBenefitSystem + + target = Path(tempfile.mkdtemp(prefix="medicaid_031_fixes_")) + for path in (AUDIT_0928 / "fixes").glob("*.py"): + shutil.copy2(path, target / path.name) + shutil.copy2(AUDIT_0922 / "fixes/r19_irs_sales_tax_2025.json", target) + spec = importlib.util.spec_from_file_location( + "latest_final", target / "latest_final.py" + ) + module = importlib.util.module_from_spec(spec) + sys.modules["latest_final"] = module + spec.loader.exec_module(module) + return CountryTaxBenefitSystem(reform=module.reform) + + +def _scalar(value): + array = np.asarray(value) + if array.dtype.kind in "OUS": + return str(array[0]) + if array.dtype == bool: + return bool(array[0]) + return float(array[0]) + + +def main() -> None: + from policyengine_us import Simulation + + from policybench.scenarios import scenario_from_dict + + scenarios = pd.read_csv(RUN / "scenarios.csv") + row = scenarios[scenarios["scenario_id"] == SCENARIO].iloc[0] + scenario = scenario_from_dict(json.loads(row["scenario_json"])) + situation = scenario.to_pe_household() + assert list(situation["people"]) == ["head"], situation["people"].keys() + + system = _system() + p = system.parameters(f"{PERIOD}-01-01") + sod = p.gov.hhs.medicaid.eligibility.categories.senior_or_disabled + parameters = { + "senior_or_disabled.income.disregard.individual[CA] (monthly)": float( + sod.income.disregard.individual[STATE] + ), + "senior_or_disabled.income.limit.individual[CA] (share of FPG)": float( + sod.income.limit.individual[STATE] + ), + "senior_or_disabled.assets.limit.individual[CA]": float( + sod.assets.limit.individual[STATE] + ), + "ssa.ssi.income.exclusions.general (monthly)": float( + p.gov.ssa.ssi.income.exclusions.general + ), + "ssa.ssi.eligibility.aged_threshold": float( + p.gov.ssa.ssi.eligibility.aged_threshold + ), + "ssa.ssi.eligibility.resources.limit.individual": float( + p.gov.ssa.ssi.eligibility.resources.limit.individual + ), + "ssa.ssi.income.sources.unearned": list(p.gov.ssa.ssi.income.sources.unearned), + "ssa.ssi.eligibility.resources.countable": list( + p.gov.ssa.ssi.eligibility.resources.countable + ), + "hhs.fpg.first_person[CONTIGUOUS_US] 2026": float( + p.gov.hhs.fpg.first_person["CONTIGUOUS_US"] + ), + "hhs.fpg.first_person[CONTIGUOUS_US] 2025": float( + system.parameters("2025-01-01").gov.hhs.fpg.first_person["CONTIGUOUS_US"] + ), + } + + readings = {} + for name, inputs in READINGS.items(): + case = copy.deepcopy(situation) + for variable, value in inputs.items(): + case["people"]["head"][variable] = {PERIOD: value} + sim = Simulation(tax_benefit_system=system, situation=case) + values = {} + for variable in PERSON_VARIABLES: + result = sim.calculate(variable, YEAR) + if hasattr(result, "decode_to_str"): + values[variable] = str(result.decode_to_str()[0]) + else: + values[variable] = _scalar(result) + readings[name] = values + + modeled = readings["modeled"] + disregard = parameters["senior_or_disabled.income.disregard.individual[CA] (monthly)"] + general = parameters["ssa.ssi.income.exclusions.general (monthly)"] + share = parameters["senior_or_disabled.income.limit.individual[CA] (share of FPG)"] + fpg = parameters["hhs.fpg.first_person[CONTIGUOUS_US] 2026"] + asset_limit = parameters["senior_or_disabled.assets.limit.individual[CA]"] + gross = modeled["ssi_marital_unearned_income"] + countable = modeled["medicaid_optional_senior_or_disabled_countable_income"] + limit = modeled["medicaid_optional_senior_or_disabled_income_limit"] + + # The reference is the published one. + reference = pd.read_csv(RUN / "reference_outputs.csv") + published = reference[ + (reference["scenario_id"] == SCENARIO) & (reference["variable"] == OUTPUT) + ] + assert len(published) == 1, published + published_value = float(published.iloc[0]["value"]) + assert published_value == 1.0, published_value + assert modeled["is_medicaid_eligible"] is True + + # Unearned income is Social Security, the pension and the IRA distribution. + parts = ( + modeled["social_security"] + + modeled["pension_income"] + + modeled["taxable_ira_distributions"] + ) + assert abs(gross - parts) < CENT, (gross, parts) + assert modeled["ssi_marital_earned_income"] == 0.0 + for variable in ( + "ssi_in_kind_support_and_maintenance", + "medicaid_optional_senior_or_disabled_unearned_income_deemed_from_ineligible_parent", + "medicaid_optional_senior_or_disabled_income_deemed_from_ineligible_spouse", + ): + assert modeled[variable] == 0.0, variable + assert modeled["ssi_couple_computation_applies"] is False + + # The CA disregard replaces the $20 general exclusion; it is not added to it. + assert disregard == 230.0 and general == 20.0 + assert abs(countable - (gross - 12 * disregard)) < CENT, (countable, gross) + assert abs(limit - share * fpg) < CENT, (limit, share, fpg) + assert share == 1.38 and fpg == 15_960.0 + assert countable <= limit + assert gross > limit + assert gross - 12 * general > limit + + # Assets and category. + assert modeled["ssi_countable_resources"] == 4_200.0 + assert asset_limit == 130_000.0 + assert modeled["ssi_countable_resources"] < asset_limit + assert modeled["is_ssi_aged"] is True and modeled["is_ssi_disabled"] is False + assert modeled["is_ssi_recipient_for_medicaid"] is False + assert modeled["is_adult_for_medicaid_nfc"] is False + assert modeled["medicaid_category"] == "SENIOR_OR_DISABLED" + # No status is listed, so the default applies. + assert modeled["immigration_status"] == "CITIZEN" + assert modeled["is_medicaid_immigration_status_eligible"] is True + + # The modeled Part B premium does not enter the test. + assert modeled["medicare_enrolled"] is True + assert abs(modeled["medicare_part_b_premium"] - 2_434.80) < CENT + for name in ("no_part_b", "not_enrolled"): + other = readings[name] + assert other["medicare_part_b_premium"] == 0.0, name + for variable in ( + "medicaid_optional_senior_or_disabled_countable_income", + "medicaid_optional_senior_or_disabled_income_limit", + "is_optional_senior_or_disabled_income_eligible", + "is_medicaid_eligible", + "medicaid_category", + ): + assert other[variable] == modeled[variable], (name, variable) + + summary = { + "gross_unearned_income": round(gross, 2), + "disregard_annual": 12 * disregard, + "countable_income": round(countable, 2), + "income_limit": round(limit, 2), + "margin_under_limit": round(limit - countable, 2), + "gross_over_limit": round(gross - limit, 2), + "gross_less_ssi_general_exclusion_only": round(gross - 12 * general, 2), + "gross_share_of_2026_fpg": round(gross / fpg, 4), + "limit_at_2025_fpg": round( + share * parameters["hhs.fpg.first_person[CONTIGUOUS_US] 2025"], 2 + ), + "monthly": { + "gross": round(gross / 12, 2), + "countable": round(countable / 12, 2), + "limit": round(limit / 12, 2), + }, + } + payload = { + "note": ( + "scenario_031 head_medicaid_eligible on the reference system " + "(policyengine-us 2.15.17 + reference_audit/2026-09-28/fixes/latest_final.py). " + "Written by scripts/engine_values.py, which asserts every relation the " + "rewritten annotations state." + ), + "policyengine_us": version("policyengine-us"), + "published_reference": published_value, + "parameters": parameters, + "summary": summary, + "readings": readings, + } + out = HERE / "verification/engine_values.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=1, sort_keys=False) + "\n") + print(json.dumps(summary, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/scripts/payload_diff.py b/reference_audit/2026-10-05-medicaid-031-annotations/scripts/payload_diff.py new file mode 100644 index 00000000..f9df8711 --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/scripts/payload_diff.py @@ -0,0 +1,133 @@ +"""Show that the rewrites change no published score: rebuild the payload both ways and diff. + +Copies the frozen run's reference files and predictions into two scratch bundles (it never +writes the snapshot), one with the annotation files as committed at ``--base`` and one with +the working tree's, and runs ``policybench.full_run_export.export_country`` on each, the +export the release drivers run. It writes ``verification/payload_diff.json``: every payload +path that differs between the two rebuilds, and every path where the base rebuild differs +from the frozen payload (data.json.gz). + + uv run python reference_audit/2026-10-05-medicaid-031-annotations/scripts/payload_diff.py \\ + --base origin/main --scratch +""" + +from __future__ import annotations + +import argparse +import gzip +import json +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +HERE = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +RUN_NAME = "us_full_run_20260612_policyengine_4_16_1_populace" +RUN = ROOT / "paper/snapshot/20260501/runs" / RUN_NAME +ANNOTATIONS = f"annotations/{RUN_NAME}" +RUN_FILES = ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", + "predictions.csv.gz", +) +ANNOTATION_FILES = ( + "us_audit_row_annotations.csv", + "us_case_notes.csv", + "us_case_reference_explanations.csv", +) + + +def _diff(a, b, path, out) -> None: + if isinstance(a, dict) and isinstance(b, dict): + for key in sorted(set(a) | set(b), key=str): + _diff(a.get(key, ""), b.get(key, ""), [*path, key], out) + elif isinstance(a, list) and isinstance(b, list) and len(a) == len(b): + for i, (x, y) in enumerate(zip(a, b)): + _diff(x, y, [*path, i], out) + elif a != b and not (isinstance(a, float) and isinstance(b, float) and a != a and b != b): + out.append(path) + + +def main() -> None: + from policybench.full_run_export import committed_reference_digest, export_country + + parser = argparse.ArgumentParser() + parser.add_argument("--base", default="origin/main") + parser.add_argument("--scratch", required=True, type=Path) + args = parser.parse_args() + + digest = committed_reference_digest("us") + payloads = {} + for label in ("base", "working_tree"): + bundle = args.scratch / label + shutil.rmtree(bundle, ignore_errors=True) + (bundle / "us").mkdir(parents=True) + (bundle / "annotations").mkdir() + for name in RUN_FILES: + shutil.copy2(RUN / name, bundle / "us" / name) + for name in ANNOTATION_FILES: + target = bundle / "annotations" / name + if label == "base": + target.write_bytes( + subprocess.run( + ["git", "-C", str(ROOT), "show", f"{args.base}:{ANNOTATIONS}/{name}"], + capture_output=True, + check=True, + ).stdout + ) + else: + shutil.copy2(ROOT / ANNOTATIONS / name, target) + payloads[label] = export_country(bundle / "us", reference_digest=digest) + + changed: list[list] = [] + _diff(payloads["base"], payloads["working_tree"], [], changed) + frozen = json.loads(gzip.open(RUN / "data.json.gz").read()) + reproduction: list[list] = [] + _diff(frozen, payloads["base"], [], reproduction) + stats = payloads["base"]["modelStats"] + + def label(path): + if path and path[0] == "modelStats": + return ["modelStats", stats[path[1]]["model"], *path[2:]] + return path + + sections = sorted({str(p[0]) for p in changed}) + cells = sorted({(p[1], p[2]) for p in changed if p[0] == "scenarioPredictions"}) + fields = sorted({str(p[-1]) for p in changed}) + record = { + "note": ( + "Payload rebuilt from the frozen run by export_country with the base's " + "annotation files and with the working tree's. 'changed' lists every path " + "that differs between the two rebuilds. 'base_vs_frozen' lists every path " + "where the base rebuild differs from the frozen data.json.gz: claude-fable-5's " + "usage, which the release drivers carry from the published payload " + "(CARRIED_USAGE in scripts/finish_gpt61sol.py) because export cannot " + "recompute it." + ), + "base": args.base, + "changed_sections": sections, + "changed_cells": [list(c) for c in cells], + "changed_fields": fields, + "changed_paths": len(changed), + "changed_by_field": {f: sum(1 for p in changed if p[-1] == f) for f in fields}, + "base_vs_frozen": [[str(x) for x in label(p)] for p in reproduction], + } + out = HERE / "verification/payload_diff.json" + out.write_text(json.dumps(record, indent=1) + "\n") + print(json.dumps(record, indent=1)) + assert sections == ["scenarioPredictions"], sections + assert cells == [("scenario_031", "head_medicaid_eligible")], cells + assert set(fields) <= {"annotation", "caseAnnotation", "referenceExplanation"}, fields + fable = {i for i, s in enumerate(stats) if s["model"] == "claude-fable-5"} + assert all(p[0] == "modelStats" and p[1] in fable for p in reproduction), ( + reproduction + ) + + +if __name__ == "__main__": + main() diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/verification/engine_values.json b/reference_audit/2026-10-05-medicaid-031-annotations/verification/engine_values.json new file mode 100644 index 00000000..10d5ceb0 --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/verification/engine_values.json @@ -0,0 +1,157 @@ +{ + "note": "scenario_031 head_medicaid_eligible on the reference system (policyengine-us 2.15.17 + reference_audit/2026-09-28/fixes/latest_final.py). Written by scripts/engine_values.py, which asserts every relation the rewritten annotations state.", + "policyengine_us": "2.15.17", + "published_reference": 1.0, + "parameters": { + "senior_or_disabled.income.disregard.individual[CA] (monthly)": 230.0, + "senior_or_disabled.income.limit.individual[CA] (share of FPG)": 1.38, + "senior_or_disabled.assets.limit.individual[CA]": 130000.0, + "ssa.ssi.income.exclusions.general (monthly)": 20.0, + "ssa.ssi.eligibility.aged_threshold": 65.0, + "ssa.ssi.eligibility.resources.limit.individual": 2000.0, + "ssa.ssi.income.sources.unearned": [ + "pension_income", + "social_security", + "disability_benefits", + "veterans_benefits", + "survivor_benefits", + "workers_compensation", + "unemployment_compensation", + "retirement_distributions", + "gi_cash_assistance", + "alimony_income", + "child_support_received", + "financial_assistance", + "dividend_income", + "interest_income", + "rental_income" + ], + "ssa.ssi.eligibility.resources.countable": [ + "bank_account_assets", + "stock_assets", + "bond_assets" + ], + "hhs.fpg.first_person[CONTIGUOUS_US] 2026": 15960.0, + "hhs.fpg.first_person[CONTIGUOUS_US] 2025": 15650.0 + }, + "summary": { + "gross_unearned_income": 23853.47, + "disregard_annual": 2760.0, + "countable_income": 21093.47, + "income_limit": 22024.8, + "margin_under_limit": 931.33, + "gross_over_limit": 1828.67, + "gross_less_ssi_general_exclusion_only": 23613.47, + "gross_share_of_2026_fpg": 1.4946, + "limit_at_2025_fpg": 21597.0, + "monthly": { + "gross": 1987.79, + "countable": 1757.79, + "limit": 1835.4 + } + }, + "readings": { + "modeled": { + "age": 67.0, + "is_ssi_aged": true, + "is_ssi_disabled": false, + "is_blind": false, + "is_ssi_aged_blind_disabled": true, + "is_ssi_recipient_for_medicaid": false, + "is_medicare_eligible": true, + "medicare_enrolled": true, + "medicare_part_b_premium": 2434.800048828125, + "social_security": 9445.470703125, + "pension_income": 13608.0, + "taxable_ira_distributions": 800.0, + "alimony_expense": 1165.0, + "ssi_unearned_income": 23853.470703125, + "ssi_marital_unearned_income": 23853.470703125, + "ssi_marital_earned_income": 0.0, + "ssi_in_kind_support_and_maintenance": 0.0, + "medicaid_optional_senior_or_disabled_unearned_income_deemed_from_ineligible_parent": 0.0, + "medicaid_optional_senior_or_disabled_income_deemed_from_ineligible_spouse": 0.0, + "ssi_couple_computation_applies": false, + "medicaid_optional_senior_or_disabled_countable_income": 21093.470703125, + "medicaid_optional_senior_or_disabled_income_limit": 22024.80078125, + "is_optional_senior_or_disabled_income_eligible": true, + "ssi_countable_resources": 4200.0, + "is_optional_senior_or_disabled_asset_eligible": true, + "is_optional_senior_or_disabled_for_medicaid": true, + "is_adult_for_medicaid_nfc": false, + "is_adult_for_medicaid": false, + "immigration_status": "CITIZEN", + "is_medicaid_immigration_status_eligible": true, + "medicaid_category": "SENIOR_OR_DISABLED", + "is_medicaid_eligible": true + }, + "no_part_b": { + "age": 67.0, + "is_ssi_aged": true, + "is_ssi_disabled": false, + "is_blind": false, + "is_ssi_aged_blind_disabled": true, + "is_ssi_recipient_for_medicaid": false, + "is_medicare_eligible": true, + "medicare_enrolled": true, + "medicare_part_b_premium": 0.0, + "social_security": 9445.470703125, + "pension_income": 13608.0, + "taxable_ira_distributions": 800.0, + "alimony_expense": 1165.0, + "ssi_unearned_income": 23853.470703125, + "ssi_marital_unearned_income": 23853.470703125, + "ssi_marital_earned_income": 0.0, + "ssi_in_kind_support_and_maintenance": 0.0, + "medicaid_optional_senior_or_disabled_unearned_income_deemed_from_ineligible_parent": 0.0, + "medicaid_optional_senior_or_disabled_income_deemed_from_ineligible_spouse": 0.0, + "ssi_couple_computation_applies": false, + "medicaid_optional_senior_or_disabled_countable_income": 21093.470703125, + "medicaid_optional_senior_or_disabled_income_limit": 22024.80078125, + "is_optional_senior_or_disabled_income_eligible": true, + "ssi_countable_resources": 4200.0, + "is_optional_senior_or_disabled_asset_eligible": true, + "is_optional_senior_or_disabled_for_medicaid": true, + "is_adult_for_medicaid_nfc": false, + "is_adult_for_medicaid": false, + "immigration_status": "CITIZEN", + "is_medicaid_immigration_status_eligible": true, + "medicaid_category": "SENIOR_OR_DISABLED", + "is_medicaid_eligible": true + }, + "not_enrolled": { + "age": 67.0, + "is_ssi_aged": true, + "is_ssi_disabled": false, + "is_blind": false, + "is_ssi_aged_blind_disabled": true, + "is_ssi_recipient_for_medicaid": false, + "is_medicare_eligible": true, + "medicare_enrolled": false, + "medicare_part_b_premium": 0.0, + "social_security": 9445.470703125, + "pension_income": 13608.0, + "taxable_ira_distributions": 800.0, + "alimony_expense": 1165.0, + "ssi_unearned_income": 23853.470703125, + "ssi_marital_unearned_income": 23853.470703125, + "ssi_marital_earned_income": 0.0, + "ssi_in_kind_support_and_maintenance": 0.0, + "medicaid_optional_senior_or_disabled_unearned_income_deemed_from_ineligible_parent": 0.0, + "medicaid_optional_senior_or_disabled_income_deemed_from_ineligible_spouse": 0.0, + "ssi_couple_computation_applies": false, + "medicaid_optional_senior_or_disabled_countable_income": 21093.470703125, + "medicaid_optional_senior_or_disabled_income_limit": 22024.80078125, + "is_optional_senior_or_disabled_income_eligible": true, + "ssi_countable_resources": 4200.0, + "is_optional_senior_or_disabled_asset_eligible": true, + "is_optional_senior_or_disabled_for_medicaid": true, + "is_adult_for_medicaid_nfc": false, + "is_adult_for_medicaid": false, + "immigration_status": "CITIZEN", + "is_medicaid_immigration_status_eligible": true, + "medicaid_category": "SENIOR_OR_DISABLED", + "is_medicaid_eligible": true + } + } +} diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/verification/independent_review.md b/reference_audit/2026-10-05-medicaid-031-annotations/verification/independent_review.md new file mode 100644 index 00000000..02a08e71 --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/verification/independent_review.md @@ -0,0 +1,96 @@ +**Verdict: APPROVE WITH NITS.** The engine mechanism is right, and so are the figures, the class counts and the model descriptions. The caveat is that I could not run the tests, the scripts or `git diff`. This session only had read-only file tools (no shell, no Write). I checked those items by reading the code and the committed records instead; the limits are listed at the end. + +## Blocking findings + +None. + +## Should fix (not blocking) + +**1. A third copy of the wrong mechanism sits in tracked code, and the README doesn't mention it.** +- README:43 says the judge's prompt "stated the wrong mechanism twice": the grounding line and the old reference explanation. +- There is a third copy. The judge instructions use it as their example of a good diagnosis, so it appears in every case's prompt (`prompt.md` line 5). It comes from `policybench/audit.py:295-297`: *"never applied the aged/disabled income test, which deducts the Medicare Part B premium from countable income"*. +- `tests/test_audit.py:665-669` repeats it as a passing example. +- I checked the annotations: no other `*_medicaid_eligible` row or case note relies on a Part B deduction. So the published damage is limited to this cell, but future audit stages will keep priming judges with the false rule. +- **Fix:** add it to "Where the old mechanism came from". Either change the example in `audit.py` and `test_audit.py` now (for instance "…which applies the state's income disregard"), or split that change into its own task with a link. + +**2. The README cites a file that doesn't exist.** +- README:83 cites `verification/independent_review.md`, but the file is not at HEAD. +- **Fix:** commit this review as that file, or drop the line. + +**3. claude-sonnet-4.6's subtype doesn't match a near-identical row.** +- claude-sonnet-4.6 is `taxable_income_or_deductions`, but glm-5.3 is `categorical_eligibility`. +- The new sonnet-4.6 text says *"its explanation settled on the expansion test, with gross income of $23,853 against $20,783"*. Its response agrees: *"Given that PolicyEngine typically implements the standard ACA Medi-Cal expansion at 138% FPL and the income exceeds that threshold…"* +- glm-5.3 made the same error (expansion limit, gross income, understated 138%) and is classed `categorical_eligibility`. +- Under the old mechanism, sonnet-4.6's income-counting subtype rested on the missed Part B deduction. Under the corrected mechanism, its main error is choosing the wrong pathway. +- **Fix:** reclassify it to `categorical_eligibility`, making the counts 28/4/3/8; no score changes. Or state in README:62 why it stays. +- gpt-5.5 (tested against a "spend-down standard") is a weaker version of the same question. + +**4. A manifest note is now false for this cell.** +- `manifest.json:24` says developer adjudications are applied before export *"so the published payload and the frozen annotations agree."* +- For this cell they no longer agree until a data release re-exports. README:71 says so; the manifest doesn't. +- **Fix:** add a sentence to the manifest note, or list the divergence somewhere a release driver checks. + +## Nits + +- **"Holds only if the disregard is skipped" overstates** (rows gemini-3.7-flash, gpt-5.6-luna, gpt-5.6-sol). Against the engine's limit, income stays over with any monthly disregard below about $152.39, SSI's $20 included. Two of these models also named no threshold. Suggested wording: "holds against the engine's limit only if the $230 disregard is skipped or replaced by a smaller one, such as SSI's $20." +- **gemini-3.5-flash:** "*elsewhere it cited gross income of $23,853*". That figure isn't in this output's explanation (*"The head's monthly income exceeds the 138% Federal Poverty Level threshold…"*). It presumably comes from another output in `raw_response`, which I could not read. Name the source ("in its SNAP explanation"), or drop the clause. +- **deepseek-v4-pro:** "about $1,757.79" puts "about" before an exact figure. +- **claude-fable-5:** "applied no disregard … and said even SSI-style disregards would leave it over" reads as self-contradictory. Try: "assumed no specific disregard, saying even SSI-style disregards would leave it over." +- **Reference explanation:** "within the limit of 138% of the 2026 federal poverty guideline for one person, $22,024.80" reads as if the guideline itself is $22,024.80. Try: "within the $22,024.80 limit (138% of the 2026 guideline for one person)." +- **Case note:** "or missed that age alone gives the head a category" also covers glm-5.2 and grok-4.3. Those two ruled the head out on Medicare grounds, so "or treated Medicare eligibility or the lack of SSI or disability as disqualifying" is more accurate. +- **README:17 is incomplete.** `is_medicaid_eligible` also ANDs the federal and Arkansas work-requirement gates, and ORs in `ca_ffyp_eligible` and `il_hbi_eligible` (`is_medicaid_eligible.py:46-54`). Neither changes anything for this head: the work requirement applies only to the ADULT and 1115 categories. Add "(plus work-requirement gates that don't apply here)". +- **README:62:** "compared income with no exclusion or disregard with the limit" uses "with" twice. Try: "compared income, with no exclusion or disregard, against the limit." +- **`engine_values.py`** never records `immigration_status`, so the "default CITIZEN" claim isn't backed by the record. The code supports it: `immigration_status_str.default_value = CITIZEN`, which feeds `immigration_status`. Add it to `PERSON_VARIABLES`. +- **`payload_diff.py:126`** checks `{p[1] for p in reproduction}` without first checking `p[0] == "modelStats"`. A difference in another section whose index happened to equal claude-fable-5's would pass unnoticed. Assert `p[0] == "modelStats"` too. +- **The test doesn't pin subtypes.** `test_rewritten_rows_keep_their_classes` checks `failure_source` and `reference_suspect` but not `failure_subtype`, which the README says is unchanged. Pin the 29/4/3/7 tally, or the per-row subtype. +- **`KEYS`/`FIELDS` are duplicated** between `apply_rewrites.py` and the test. This is minor drift risk. +- **The release drivers don't re-apply the ledger.** The next release that rebuilds annotations from stage verdicts will fail the test until someone runs `apply_rewrites.py` by hand. That is acceptable as a tripwire, but a one-line note in the release driver or its docs would help. + +## What I checked and found correct + +**Engine (policyengine-us 2.15.17 source):** +- **The $230 replaces the $20; it is not added.** `_apply_ssi_exclusions.py:26`: `general_exclusion = p.general if general_exclusion is None else general_exclusion`. `_apply_medicaid_optional_senior_or_disabled_exclusions.py:73-79` passes `general_exclusion=income_disregard` for every state except CT and MO. +- **Disregard:** CA = 230 per month (`disregard/individual.yaml:26-27`). +- **Income limit and test:** CA = 1.38 from 2021 (`limit/individual.yaml:29-31`), applied as `limit_pct * unit_fpg`. The test is `income <= income_limit`, so "at or under" is right. +- **Asset test:** CA = 130,000 from 2026-01-01, tested as `assets < asset_limit`, so "below" is right. +- **Countable income formula:** it adds `ssi_marital_unearned_income`, income deemed from an ineligible parent, and ISM. Spousal deeming is added after the disregard. All of this matches README:19-24. +- **Category:** `is_optional_senior_or_disabled_for_medicaid` is aged/blind/disabled AND income AND assets. `is_ssi_aged` is `age >= aged_threshold`, which is 65. +- **Medicare bar:** `is_adult_for_medicaid_nfc` is `age_eligible & ~medicare_eligible & ~ssi_excluded`. `medicaid_work_requirement_eligible` uses Medicare as an exemption. I spot-checked the parent, young-adult, pregnant, medically-needy, SSI-recipient and immigration formulas plus the buy-in and 1115 categories: none reads `is_medicare_eligible`. I could not grep the whole directory (the venv's `.gitignore` hides it from ripgrep). +- **Immigration default:** CITIZEN, via `immigration_status_str`. + +**Arithmetic:** +- $23,853.47 − $2,760 = $21,093.47. +- 1.38 × $15,960 = $22,024.80, or $1,835.40 a month. +- Margin under the limit: $931.33. Gross over the limit: $1,828.67. Gross less only the $20 exclusion: $23,613.47. +- Gross is about 149% of the 2026 guideline. +- 1.38 × $15,650 = $21,597 (the 2025 figure). +- 1.38 × $15,060 = $20,783 (sonnet-4.6's figure). + +All of these agree with the committed `engine_values.json`. + +**Model descriptions:** I checked all 43, not just a sample, against each model's explanation as embedded in the stage judge prompt (`results/local/gpt61sol-stage/.../us__scenario_031__head_medicaid_eligible/prompt.md:76-203`). Every quote and figure matches, apart from the gemini-3.5-flash clause above. Examples: +- claude-fable-5: ~$1,890 a month, ~$1,800 limit. +- glm-5.3: "~199% FPL", ~$1,600. +- qwen-3.7-max: "typical aged/blind/disabled Medicaid limits". +- claude-opus-4.8: "well below the threshold. Wait—…100% FPL". + +**Classes:** +- All 43 rows are `llm_error` with `reference_suspect=False`. +- The subtype counts are 29/4/3/7, matching the README. +- The README's 17/2/2/8 split of the 29 checks out row by row. +- `rewrites.json` carries only text fields, and the test enforces that. + +**Tooling:** +- `apply_rewrites.py` refuses a missing or non-unique key, refuses a text that is neither old nor new, and requires a byte-for-byte CSV and manifest round-trip. +- `--check` writes nothing and fails on stale hashes. +- The manifest pins the three files under `audit_annotation_artifacts.files`. +- `payload_diff.json` shows only the scenario_031 cell's three text fields changing (43 + 43 + 46 paths). The base rebuild matches the frozen `data.json.gz` except four carried claude-fable-5 usage fields. That is good evidence that no score changes, and `paper_results.py` reads only classes from these files. + +## Not verified: rerun before merge + +I could not execute anything, so these are still open: +- the pytest command and `apply_rewrites.py --check`; +- the `engine_values.py` rerun and the `git diff` of its output; +- `git diff origin/main` (that only the 45 fields and 3 hashes changed, and the frozen run dir is untouched); +- `raw_response` in `predictions.csv.gz`, which is gzipped. I read model behaviour from the explanation text in the judge prompt instead. +- the README:62 corpus tally ("97 rows… one `state_local_rule`"). \ No newline at end of file diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/verification/payload_diff.json b/reference_audit/2026-10-05-medicaid-031-annotations/verification/payload_diff.json new file mode 100644 index 00000000..9823a1e7 --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/verification/payload_diff.json @@ -0,0 +1,46 @@ +{ + "note": "Payload rebuilt from the frozen run by export_country with the base's annotation files and with the working tree's. 'changed' lists every path that differs between the two rebuilds. 'base_vs_frozen' lists every path where the base rebuild differs from the frozen data.json.gz: claude-fable-5's usage, which the release drivers carry from the published payload (CARRIED_USAGE in scripts/finish_gpt61sol.py) because export cannot recompute it.", + "base": "origin/main", + "changed_sections": [ + "scenarioPredictions" + ], + "changed_cells": [ + [ + "scenario_031", + "head_medicaid_eligible" + ] + ], + "changed_fields": [ + "annotation", + "caseAnnotation", + "referenceExplanation" + ], + "changed_paths": 132, + "changed_by_field": { + "annotation": 43, + "caseAnnotation": 43, + "referenceExplanation": 46 + }, + "base_vs_frozen": [ + [ + "modelStats", + "claude-fable-5", + "costPerHousehold" + ], + [ + "modelStats", + "claude-fable-5", + "costUsd" + ], + [ + "modelStats", + "claude-fable-5", + "latencySeconds" + ], + [ + "modelStats", + "claude-fable-5", + "totalTokens" + ] + ] +} diff --git a/reference_audit/2026-10-05-medicaid-031-annotations/verification/row_review.json b/reference_audit/2026-10-05-medicaid-031-annotations/verification/row_review.json new file mode 100644 index 00000000..fbd6bd08 --- /dev/null +++ b/reference_audit/2026-10-05-medicaid-031-annotations/verification/row_review.json @@ -0,0 +1,646 @@ +{ + "note": "Review record for the rewritten row annotations. Each model's row was drafted, then checked by a separate agent that read the model's frozen response (predictions.csv.gz raw_response and explanation) and the engine fact sheet, quoted the response words each claim rests on, and corrected the draft. A final critic checked consistency across rows and against the case note. The final text of each row is in ../rewrites.json; it differs from 'checked_text' only by the edits listed in 'post_review_edits'.", + "post_review_edits": [ + "Inserted '(,760 a year)' after ' monthly disregard' in 21 rows that subtracted the monthly disregard from an annual figure (critic), and merged the resulting double parentheses.", + "grok-build-0.1: 'aged Medi-Cal' -> 'aged Medicaid'; gemini-3.1-flash-lite-preview: 'the Medi-Cal limit' -> 'the Medicaid limit'.", + "glm-5.2 and grok-4.3: 'The engine has no Medicare exclusion' -> 'In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group' (in variables/gov/hhs/medicaid, is_adult_for_medicaid_nfc is the only formula that bars on is_medicare_eligible; medicaid_work_requirement_eligible reads it as an exemption).", + "After the independent review (verification/independent_review.md): gemini-3.7-flash, gpt-5.6-luna and gpt-5.6-sol now say income is over the limit only if the disregard is skipped or replaced by a smaller one, such as SSI's $20; gemini-3.5-flash names its federal tax and SNAP answers as the source of $23,853 and drops 'about' before exact figures; deepseek-v4-pro drops 'about'; claude-fable-5 is reworded ('assumed no specific disregard')." + ], + "tally": { + "total": 43, + "buckets": [ + { + "bucket": "gross income vs the limit (stated gross or total income, no exclusion or disregard)", + "count": 17, + "models": [ + "claude-fable-5 (gross less alimony)", + "claude-haiku-4.5", + "claude-opus-5", + "claude-opus-5.5", + "claude-sonnet-4.6", + "deepseek-v4-flash-0731", + "deepseek-v4-pro", + "deepseek-v4-pro-0813", + "deepseek-v4.1-flash", + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.6-flash", + "gemini-3.8-flash", + "gpt-5.5", + "grok-4.6", + "minimax-m3", + "qwen-3.7-max" + ] + }, + { + "bucket": "only SSI's $20 exclusion", + "count": 2, + "models": [ + "claude-fable-5.1", + "grok-4.7" + ] + }, + { + "bucket": "a general income exclusion of unstated size (most likely SSI's $20)", + "count": 2, + "models": [ + "gpt-6-astra", + "gpt-6.1-sol" + ] + }, + { + "bucket": "wrong threshold", + "count": 4, + "models": [ + "claude-opus-4.8 (100%, $15,650)", + "inkling (100%, $15,650)", + "claude-sonnet-5 (SSI income limit, $13,596)", + "gpt-5.6-terra (SSI-linked income level)" + ] + }, + { + "bucket": "asset error", + "count": 3, + "models": [ + "grok-build-0.1 ($2,000 SSI limit, only reason)", + "ox-alpha ($2,000 SSI limit, plus gross income)", + "grok-4.5 (unstated resource limit, plus gross income)" + ] + }, + { + "bucket": "category error (Medicare bar, required disability/blindness/SSI, no category, or adult group only)", + "count": 7, + "models": [ + "glm-5.2 (Medicare bars Medicaid)", + "grok-4.3 (dual eligibles excluded)", + "kimi-k2.6 (not disabled, blind or SSI)", + "qwen3.8-max (not disabled, blind or in a category)", + "gpt-5.4-nano (no disability or low-income flag)", + "claude-sonnet-5.5 (no categorical group, plus gross $23.9k)", + "glm-5.3 (adult expansion test only)" + ] + }, + { + "bucket": "no derivation (no income figure; asserted income too high)", + "count": 8, + "models": [ + "gemini-3.1-flash-lite-preview", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.7-flash", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-6-sol", + "kimi-k3" + ] + }, + { + "bucket": "other", + "count": 0, + "models": [] + } + ], + "subtype_crosscheck": "Income-counting buckets 17 + 2 + 2 + 8 = 29, which equals the 29 taxable_income_or_deductions rows. Threshold 4 = thresholds_rates 4, asset 3 = asset_resource 3, category 7 = categorical_eligibility 7. Total 43. So 'most models went wrong on countable income' holds (29 of 43).", + "secondary_errors": "These rows also compared gross income with a limit outside their primary bucket: claude-opus-4.8, inkling, claude-sonnet-5.5, grok-4.5, ox-alpha, glm-5.3. Counting them, 23 rows stated or used gross income, and 25 with the two $20-only rows. Each of these understated limits would also have produced 0 against the engine's $21,093.47: claude-haiku-4.5 ($18,564), deepseek-v4-pro ($1,732 a month), claude-sonnet-4.6 ($20,783), glm-5.3 ($1,600 a month). Alimony was subtracted by claude-fable-5 and claude-haiku-4.5. Assets were also doubted by claude-haiku-4.5.", + "boundary_calls": "gemini-3.8-flash is in the gross bucket because it said 'total income', though it gave no figure. gemini-3.5-flash is in no derivation because it named 138% FPL but gave no income figure. minimax-m3 is in the gross bucket because it guessed about $24k against an unstated limit. claude-sonnet-5.5 and qwen-3.7-max both gave gross figures and a category reason; they are split by their subtypes.", + "parser_contingency": "claude-opus-4.8 and claude-sonnet-4.6 submitted a structured value of 1 (verified in raw_response this session). If they are rescored as correct, the total is 41: gross 16 and threshold 3. The corrected closing summary still holds unchanged (28 of 41 income-counting)." + }, + "rows": [ + { + "model": "claude-fable-5", + "verdict": "fixed", + "problems_found": [ + "raw_response is null; the check rests on the stored explanation alone.", + "The draft's 'but it subtracted the $1,165 of alimony' presents the alimony subtraction as the error that pushed income over the limit. That subtraction lowered income, which helped the model. The deciding error was that it applied no disregard: $22,688 / 12 = $1,890.67, so no exclusion was taken. Even its own $1,890 less California's $230 would be about $1,660, under its $1,800. The rewrite now names the missing disregard as the error and keeps alimony as a side note.", + "'Said SSI-style disregards would not change the result' is tightened to the model's actual claim ('even with SSI-style disregards ... exceeds'). Engine figures checked: $1,987.79, $1,757.79 and $1,835.40 are correct, and monthly figures fit because the model worked monthly." + ], + "evidence": "\"less alimony expense adjustment $1,165 gives MAGI-style income ~$22,688; even with SSI-style disregards, monthly income (~$1,890) exceeds the 138% FPL limit for one person (~$1,800/month in 2026)\"; \"eligibility would come via the aged/senior pathway (ACA MAGI expansion covers only ages 19-64)\"", + "checked_text": "It took the aged pathway and a 138% limit of about $1,800 a month but applied no disregard: it subtracted the $1,165 of alimony, put income at about $1,890 a month, and said even SSI-style disregards would leave it over. The engine does not subtract alimony; it takes $1,987.79 a month less California's $230 disregard, which replaces SSI's $20 exclusion, for $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline).", + "final_text": "It took the aged pathway and a 138% limit of about $1,800 a month, subtracted the $1,165 of alimony to put income at about $1,890 a month, and assumed no specific disregard, saying even SSI-style disregards would leave it over. The engine does not subtract alimony; it takes $1,987.79 a month less California's $230 disregard, which replaces SSI's $20 exclusion, for $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. The income count decided the case: with $1,757.79 it would pass even its own $1,800 limit." + }, + { + "model": "claude-fable-5.1", + "verdict": "ok", + "problems_found": [ + "None found. The $240 exclusion, $23,613, the aged pathway and the ~$21,600 limit all match the explanation. Arithmetic checks out: $23,853.47 - $2,760 = $21,093.47, which is under $22,024.80. The text states that the disregard replaces the $20 exclusion. Minor: the exclusion is called '$240 annual' and then 'that $20 exclusion'. The two are consistent ($20 a month) and need no change." + ], + "evidence": "\"under the CA aged/disabled pathway, countable income of about $23,613/year (23,853 minus $240 exclusion) exceeds the 138% FPL limit (~$21,600), so not eligible for Medi-Cal\"", + "checked_text": "It used the aged pathway but applied only SSI's $240 annual general exclusion, putting countable income at $23,613 against a 138% limit of about $21,600. California's $230 monthly disregard takes the place of that $20 exclusion, so the engine counts $23,853.47 less $2,760, or $21,093.47, which is under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It used the aged pathway but applied only SSI's $240 annual general exclusion, putting countable income at $23,613 against a 138% limit of about $21,600. California's $230 monthly disregard takes the place of that $20 exclusion, so the engine counts $23,853.47 less $2,760, or $21,093.47, which is under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Missing the CA disregard decided the case: $21,093.47 passes even its $21,600 limit." + }, + { + "model": "claude-haiku-4.5", + "verdict": "fixed", + "problems_found": [ + "'Compared income of $23,853, with or without the $1,165 of alimony' is muddled, since $23,853 less alimony is $22,688. Reworded to say it tested gross $23,853 and that amount less alimony.", + "Added 'gross' so the text shows that no exclusion or disregard was applied, which matches the response.", + "The engine statements are correct: alimony is not subtracted, $21,093.47 < $22,024.80, and the $130,000 asset limit is right. The asset doubt is quoted accurately." + ], + "evidence": "\"approximately 138% of federal poverty line (around $1,547/month or ~$18,564 annually). Head's total income of $23,853 exceeds this threshold. Additionally, after accounting for deductible items (alimony expense of $1,165), countable income remains above the limit. With bank account assets of $4,200, which may exceed resource limits\"", + "checked_text": "It compared gross income of $23,853, and that amount less the $1,165 of alimony, against a 138% limit it understated at about $18,564, and doubted the $4,200 in assets. The engine subtracts no alimony: $23,853.47 less California's $230 monthly disregard, which replaces SSI's $20 exclusion, leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline). The $4,200 is far under California's $130,000 asset limit.", + "final_text": "It compared gross income of $23,853, and that amount less the $1,165 of alimony, against a 138% limit it understated at about $18,564, and doubted the $4,200 in assets. The engine subtracts no alimony: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline). The $4,200 is far under California's $130,000 asset limit.", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Gross income and the understated limit each force 0 on their own, so the income-count subtype is not clearly wrong and follows corpus practice." + }, + { + "model": "claude-opus-4.8", + "verdict": "fixed", + "problems_found": [ + "'Concluded the head was ineligible' overstates what the model submitted. The structured value in its Medicaid tool call is 1, and only the explanation trailer says 0. Reworded to 'its explanation concluded'.", + "PARSER FLAG CONFIRMED THIS SESSION: running policybench.reparse_predictions.parse_serialized_response (repo at c7aaffe9) on this raw_response returns head_medicaid_eligible = 1.0 and logs 'Structured value disagrees with explanation trailer ... keeping structured value'. The recorded prediction is 0. Under the current parser the row would score as correct and would not be an llm_error row. The orchestrator must decide before this annotation ships.", + "The engine figures check out: 138% of the 2026 guideline is $22,024.80, and $23,853.47 - $2,760 = $21,093.47. $15,650 is the 2025 guideline, per the fact sheet." + ], + "evidence": "Tool call {\"value\": 1, ...}: \"Countable income: Social Security $9,445 + IRA $800 + pension $13,608 = $23,853, well below the threshold. Wait—MAGI for aged/disabled in PolicyEngine uses 100% FPL-based limits for a single person (~$15,650), so $23,853 exceeds it. value = 0\"", + "checked_text": "It first called gross income of $23,853 well below the threshold, then switched to a MAGI-based 100% limit of about $15,650 (the 2025 guideline), and its explanation concluded the head was ineligible. The engine's senior pathway uses SSI income rules and 138% of the 2026 guideline, $22,024.80; it counts $23,853.47 less California's $230 monthly disregard, which replaces SSI's $20 exclusion, for $21,093.47, under the limit.", + "final_text": "It first called gross income of $23,853 well below the threshold, then switched to a MAGI-based 100% limit of about $15,650 (the 2025 guideline), and its explanation concluded the head was ineligible. The engine's senior pathway uses SSI income rules and 138% of the 2026 guideline, $22,024.80; it counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the limit.", + "subtype": "thresholds_rates", + "subtype_note": "Kept. The switch to a 100% limit would fail even the engine's $21,093.47, so the threshold is decisive. The row may not be a scored error at all (see the parser flag)." + }, + { + "model": "claude-opus-5", + "verdict": "ok", + "problems_found": [ + "None found. Gross $23,853, the 100% to 138% range and $15,650 to $21,600 all match the explanation. Both figures are 2025-guideline numbers: $15,650 at 100% and $21,597 at 138%. The engine path and arithmetic are correct, and nothing about premiums remains." + ], + "evidence": "\"Annual countable income ~$23,853 (including Social Security) is above the aged/ABD Medi-Cal limit (~100-138% FPL for one person, roughly $15,650-$21,600), so not eligible.\"", + "checked_text": "It tested gross income of $23,853 against an aged limit it gave as 100% to 138% of the poverty line, about $15,650 to $21,600 (2025-guideline figures), and found it over. The engine counts $23,853.47 less California's $230 monthly disregard, which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It tested gross income of $23,853 against an aged limit it gave as 100% to 138% of the poverty line, about $15,650 to $21,600 (2025-guideline figures), and found it over. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. The deciding error was gross income with no disregard." + }, + { + "model": "claude-opus-5.5", + "verdict": "ok", + "problems_found": [ + "None found. The 138% aged limit, about $21,600 (138% of the 2025 guideline is $21,597), and gross $23,853 match the explanation. The engine figures are correct, and the text does not fault its 138% rate." + ], + "evidence": "\"Annual income of $23,853 exceeds the aged Medi-Cal limit of 138% FPL (about $21,600), so not eligible.\"", + "checked_text": "It used the 138% aged limit, at about $21,600 (the 2025-guideline figure), but compared it with gross income of $23,853. The engine counts $23,853.47 less California's $230 monthly disregard, which replaces SSI's $20 exclusion, for $21,093.47, under its $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It used the 138% aged limit, at about $21,600 (the 2025-guideline figure), but compared it with gross income of $23,853. The engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under its $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. The only deciding error was gross income with no disregard: $21,093.47 passes even its $21,600 limit." + }, + { + "model": "claude-sonnet-4.6", + "verdict": "fixed", + "problems_found": [ + "The draft says it 'settled on gross income of $23,853 against $20,783' but does not say which test it settled on. Its final reasoning used the ACA expansion test at 138%. The rewrite says so.", + "The structured value in its Medicaid tool call is 1, and only the explanation trailer says 0. Reworded to 'its explanation settled'.", + "PARSER FLAG CONFIRMED THIS SESSION: parse_serialized_response (policybench c7aaffe9) on this raw_response returns 1.0 and keeps the structured value. The recorded prediction is 0. Under the current parser the row would score as correct, so the orchestrator must decide before this ships.", + "Arithmetic checked: 1.38 x $15,060 = $20,782.80 (about $20,783), and $15,060 is below the 2026 guideline of $15,960. The engine figures are correct." + ], + "evidence": "Tool call {\"value\": 1, ...}: \"138% FPL for a single individual is approximately $20,783/year\"; \"Aged & Disabled Federal Poverty Level (A&D FPL) program, seniors 65+ with income up to 100% FPL (~$15,060 for 2026)\"; \"Given that PolicyEngine typically implements the standard ACA Medi-Cal expansion at 138% FPL and the income exceeds that threshold, the Head is not eligible. value = 0\"", + "checked_text": "It moved between the adult expansion test, which excludes someone aged 67, and a 100% aged limit of $15,060; its explanation settled on the expansion test, with gross income of $23,853 against $20,783, which is 138% of its understated $15,060 guideline. The engine's senior pathway counts $23,853.47 less California's $230 monthly disregard, which replaces SSI's $20 exclusion, for $21,093.47, under $22,024.80 (138% of the 2026 guideline).", + "final_text": "It moved between the adult expansion test, which excludes someone aged 67, and a 100% aged limit of $15,060; its explanation settled on the expansion test, with gross income of $23,853 against $20,783, which is 138% of its understated $15,060 guideline. The engine's senior pathway counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under $22,024.80 (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Gross income with no disregard fails even the correct limit, so the income count was a necessary error. The pathway confusion does not change the 138% rate. The row may not be a scored error at all (see the parser flag)." + }, + { + "model": "claude-sonnet-5", + "verdict": "ok", + "problems_found": [ + "None found. $14,408 (pension plus IRA), the claim that 138% applies only to expansion adults, the cited $21,597, and the binding $13,596 SSI limit all match the response and its order. It correctly said assets were under the limit, and the text does not fault that. The engine claims are correct: Social Security counts in full, and $21,093.47 < $22,024.80." + ], + "evidence": "\"Social Security ($9,445, mostly excluded) ... totaling roughly $14,408\"; \"about 138% FPL is only for MAGI expansion adults under 65\"; \"Aged & Disabled Federal Poverty Level program limit is about 138% FPL (~$21,597 for single in 2026)\"; \"this exceeds the SSI income limit (~$1,133/month = $13,596/year)\"", + "checked_text": "It excluded most Social Security to get $14,408, said 138% applied only to expansion adults under 65, cited a 138% aged limit of about $21,597 that $14,408 would pass, then used an SSI limit of about $13,596 as binding. The engine counts Social Security in full and uses 138% of the 2026 guideline, $22,024.80: $23,853.47 less California's $230 monthly disregard gives $21,093.47, under the limit.", + "final_text": "It excluded most Social Security to get $14,408, said 138% applied only to expansion adults under 65, cited a 138% aged limit of about $21,597 that $14,408 would pass, then used an SSI limit of about $13,596 as binding. The engine counts Social Security in full and uses 138% of the 2026 guideline, $22,024.80: $23,853.47 less California's $230 monthly disregard ($2,760 a year) gives $21,093.47, under the limit.", + "subtype": "thresholds_rates", + "subtype_note": "Kept. Its income error ran in its favor, so the binding $13,596 SSI limit is what produced 0." + }, + { + "model": "claude-sonnet-5.5", + "verdict": "fixed", + "problems_found": [ + "'By itself meets the engine's senior-or-disabled category' is imprecise. In the engine the category also requires the income and asset tests, and age alone meets only its aged-blind-disabled condition. Reworded to 'meets the category condition of the engine's senior-or-disabled pathway'.", + "Added 'with no disregard' so the income error is stated directly. $23.9k is the gross sum, so no exclusion was applied.", + "The engine figures and arithmetic are correct, and the text does not fault its 138% rate or its inclusion of Social Security." + ], + "evidence": "\"Not eligible: countable income of about $23.9k including Social Security exceeds 138% FPL, and Head is not in a categorical group.\"", + "checked_text": "It said the head was in no categorical group and that countable income of about $23.9k, including Social Security and with no disregard, exceeded 138% FPL. At 67 the head is aged, which alone meets the category condition of the engine's senior-or-disabled pathway, and the engine counts $23,853.47 less California's $230 monthly disregard, which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It said the head was in no categorical group and that countable income of about $23.9k, including Social Security and with no disregard, exceeded 138% FPL. At 67 the head is aged, which alone meets the category condition of the engine's senior-or-disabled pathway, and the engine counts $23,853.47 less California's $230 monthly disregard ($2,760 a year), which replaces SSI's $20 exclusion, for $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "categorical_eligibility", + "subtype_note": "Kept. Both stated reasons force 0 on their own. The categorical denial is the more basic error, so the current subtype is not clearly wrong." + }, + { + "model": "deepseek-v4-flash-0731", + "verdict": "ok", + "problems_found": [ + "Rewrite checked and correct, so no change needed. The old annotation's Part B and premium mechanism and its figure of about $21,400 are gone.", + "Every model claim is supported: gross $23,853 compared with 138% FPL, no dollar limit stated, no exclusion in the Medicaid row (its SSI row did subtract $20).", + "Engine figures match the fact sheet: $2,760 disregard replacing SSI's $20 exclusion, $21,093.47 countable income, $22,024.80 limit at 138% of the 2026 guideline." + ], + "evidence": "Medicaid row: \"Head is 67 (Medicare-age) with income of about $23,853/year, exceeding the relevant income limits for aged Medicaid eligibility (over 138% FPL)\". SSI row: \"about $1,968 after the $20 general exclusion\".", + "checked_text": "It compared gross income of about $23,853 with 138% FPL and concluded the head was over the limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It compared gross income of about $23,853 with 138% FPL and concluded the head was over the limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Its 138% FPL limit was the right one, so the deciding error is counting gross income without California's disregard. That is the corpus's income-counting subtype." + }, + { + "model": "deepseek-v4-pro", + "verdict": "ok", + "problems_found": [ + "Rewrite checked and correct, so no change needed. The Part B mechanism and the old figures ($1,785 a month income, about $1,800 limit) are gone.", + "Arithmetic checks out: $1,987.75 - $230 = $1,757.75 on the model's figure ($1,757.79 on the engine's). That is under the $1,835.40 limit but above the model's $1,732, so both errors were needed for its 0. The annotation says so." + ], + "evidence": "Medicaid row: \"income of $1,987.75/month exceeds CA Aged & Disabled FPL income limit of 138% FPL ($1,732/month estimated)\". SSI row: \"Countable unearned income $1,967.75/month ($1,987.75 - $20)\".", + "checked_text": "It tested gross monthly income of $1,987.75 against an understated 138% FPL limit of $1,732. The engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves about $1,757.79, under the actual limit of $1,835.40 a month (138% of the 2026 guideline). Both errors mattered: income after the disregard still tops the model's $1,732 figure.", + "final_text": "It tested gross monthly income of $1,987.75 against an understated 138% FPL limit of $1,732. The engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $1,757.79, under the actual limit of $1,835.40 a month (138% of the 2026 guideline). Both errors mattered: income after the disregard still tops the model's $1,732 figure.", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Two errors were each necessary: the missed disregard and the understated limit. Income counting matches corpus practice and is not clearly wrong; thresholds_rates would be the secondary label." + }, + { + "model": "deepseek-v4-pro-0813", + "verdict": "ok", + "problems_found": [ + "Rewrite checked and correct, so no change needed. The Part B mechanism is gone, and so is the unsupported claim that it required SSI receipt.", + "It did call its $23,853 countable income and named an FPL-based senior limit with no dollar figure, which the rewrite says. Its remark of no SSI categorical eligibility is true in the engine (the head is not an SSI recipient), so the rewrite rightly does not fault it." + ], + "evidence": "Scored response: \"Annual countable income of about $23,853 exceeds California's applicable Medi-Cal FPL-based income limit for this senior single-person household and there is no SSI categorical eligibility\". Other response: \"income of $23,853 exceeds the Medi-Cal aged/disabled eligibility threshold\".", + "checked_text": "It treated the full $23,853 as countable income and found it over California's FPL-based limit for a senior, without stating the limit. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It treated the full $23,853 as countable income and found it over California's FPL-based limit for a senior, without stating the limit. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. The deciding error is counting gross income without the disregard. Its SSI remark was accurate and did not decide its answer." + }, + { + "model": "deepseek-v4.1-flash", + "verdict": "ok", + "problems_found": [ + "Rewrite checked and correct, so no change needed. The false claim that the count requires a Part B deduction, and the $21,400 figure, are gone.", + "Its $22,218 limit (138% of $16,100) is slightly above the engine's $22,024.80. $21,093.47 is under both, so the missed disregard decides the case, as the rewrite says." + ], + "evidence": "Medicaid row: \"Annual income of $23,853 exceeds California's aged/blind/disabled Medicaid income limit of 138% FPL ($22,218 for a single person in 2026)\".", + "checked_text": "It compared gross income of $23,853 with its own 138% FPL figure of $22,218, slightly above the engine's $22,024.80 limit (138% of the 2026 guideline). The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under either figure.", + "final_text": "It compared gross income of $23,853 with its own 138% FPL figure of $22,218, slightly above the engine's $22,024.80 limit (138% of the 2026 guideline). The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under either figure.", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. The deciding error is the missed disregard. Its slightly high limit would not have changed the answer." + }, + { + "model": "gemini-3-flash-preview", + "verdict": "ok", + "problems_found": [ + "Rewrite checked and correct, so no change needed. The Part B mechanism and the unquoted program label \"Aged & Disabled\" are gone.", + "Its about $22,080 limit is 138% of $16,000, close to the engine's $22,024.80. It tested gross $23,853 against it with no exclusion, which the rewrite says." + ], + "evidence": "Medicaid row: \"Medi-Cal for the Aged and Disabled generally uses a 138% FPL income limit. Head's annual income of $23,853 exceeds 138% of the projected FPL for 2026 (approximately $22,080 for a household of one)\".", + "checked_text": "It used a 138% FPL limit of about $22,080, close to the engine's $22,024.80 (138% of the 2026 guideline), but tested gross income of $23,853 against it. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves countable income of $21,093.47, under the limit.", + "final_text": "It used a 138% FPL limit of about $22,080, close to the engine's $22,024.80 (138% of the 2026 guideline), but tested gross income of $23,853 against it. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves countable income of $21,093.47, under the limit.", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Its threshold was essentially right, so the deciding error is counting gross income without the disregard." + }, + { + "model": "gemini-3.1-flash-lite-preview", + "verdict": "fixed", + "problems_found": [ + "The draft's \"senior-or-disabled pathway, which age 67 meets\" overstated the engine. Age meets only the pathway's aged test; the pathway also has income and asset tests. Reworded to say the head meets the aged test, which needs no SSI or disability.", + "The model framed the limit as one \"for a single adult\", not a senior limit. That is now quoted as its own words.", + "The rest is correct: the Medicaid row gives no figures or pathway, and the Part B mechanism and the old unsupported \"gross income of $23,853 with an FPL limit\" claim are gone." + ], + "evidence": "Medicaid row: \"The household income exceeds the California Medicaid (Medi-Cal) income limit for a single adult.\" Federal tax row: \"total gross income of $23,853\".", + "checked_text": "It gave no figures or pathway, saying only that household income exceeded the Medi-Cal limit \"for a single adult.\" At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard (in place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It gave no figures or pathway, saying only that household income exceeded the Medicaid limit \"for a single adult.\" At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Its only stated reason was income over the limit. The response does not clearly show a category or threshold error, so the current subtype is not clearly wrong." + }, + { + "model": "gemini-3.1-pro-preview", + "verdict": "ok", + "problems_found": [ + "Rewrite checked and correct, so no change needed. The premium and Part B mechanism and the unquoted \"Aged & Disabled\" label are gone.", + "It named 138% FPL but no dollar limit, and used gross $23,853 (its SSI row subtracted $240, its Medicaid row did not). The rewrite says this accurately and does not fault its pathway or percentage." + ], + "evidence": "Medicaid row: \"The head's income of $23,853 is above the 138% Federal Poverty Level threshold for California's Aged, Blind, and Disabled Medi-Cal program.\" SSI row: \"$23,853 total minus the $240 annual exclusion\".", + "checked_text": "It compared gross income of $23,853 with 138% FPL, without stating the dollar limit, and found it over. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It compared gross income of $23,853 with 138% FPL, without stating the dollar limit, and found it over. The engine subtracts California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. It had the right pathway and percentage, so the deciding error is counting gross income without the disregard." + }, + { + "model": "gemini-3.5-flash", + "verdict": "fixed", + "problems_found": [ + "The draft's \"consistent with the gross $23,853 it cited elsewhere\" was a hedged inference. It now states the facts: the Medicaid row gave no figures, and it cited gross income of $23,853 in its other rows.", + "The model's phrase is \"free non-MAGI Medi-Cal\"; the full quote replaces the shortened one.", + "The rest is correct: the 138% FPL limit matches the engine, and the monthly figures check out ($1,987.79 - $230 = $1,757.79, under $1,835.40). The Part B mechanism is gone." + ], + "evidence": "Medicaid row: \"The head's monthly income exceeds the 138% Federal Poverty Level threshold for free non-MAGI Medi-Cal (Medicaid) in California.\" Federal tax row: \"The head's gross income of $23,853\"; SNAP row: \"With a gross income of $23,853\".", + "checked_text": "It correctly picked the 138% FPL limit for what it called \"free non-MAGI Medi-Cal\" but gave no figures, saying only that the head's monthly income was over it; elsewhere it cited gross income of $23,853. The engine subtracts California's $230 monthly disregard from about $1,987.79 a month, in place of SSI's $20 exclusion. That leaves about $1,757.79 a month, under the $1,835.40 monthly limit (138% of the 2026 guideline).", + "final_text": "It correctly picked the 138% FPL limit for what it called \"free non-MAGI Medi-Cal\" but gave no figures, saying only that the head's monthly income was over it; in the same response it put gross income at $23,853 for its federal tax and SNAP answers. The engine subtracts California's $230 monthly disregard from $1,987.79 a month, in place of SSI's $20 exclusion. That leaves $1,757.79 a month, under the $1,835.40 monthly limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Its pathway and threshold were right, so the deciding error is the income count without the disregard." + }, + { + "model": "gemini-3.5-flash-lite", + "verdict": "fixed", + "problems_found": [ + "The draft's \"Age 67 qualifies the head for the engine's senior-or-disabled pathway\" overstated the engine. Age meets only the aged test; income and asset tests also apply. Reworded to the aged test, which needs no SSI or disability.", + "The rest is correct: no figures, threshold or pathway, only age and income cited. The Part B mechanism, the claim that it skipped a premium deduction, and the unsupported mention of assets are gone." + ], + "evidence": "Medicaid row: \"Based on the head's age and income, Medicaid eligibility is not met under California rules for this household structure.\" SSI row: \"The head's Social Security and pension income exceed the SSI income limit\".", + "checked_text": "It gave no figures, threshold or pathway, citing only the head's age and income. At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard (in place of SSI's $20 exclusion) is $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It gave no figures, threshold or pathway, citing only the head's age and income. At 67 the head meets the aged test of the engine's senior-or-disabled pathway, which needs no SSI or disability. Income of $23,853.47 less California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) is $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. It cited age and income without detail and showed no clear category error. Income counting is this case's deciding step and the corpus default, so the current subtype is not clearly wrong." + }, + { + "model": "gemini-3.6-flash", + "verdict": "ok", + "problems_found": [], + "evidence": "\"The head's annual income of $23,853 exceeds the eligibility threshold for CA Medicaid (Medi-Cal) for an individual aged 65 or older.\" (one sentence only; no figure given for the threshold, no exclusion or disregard)", + "checked_text": "It treated the full $23,853 as countable and said it exceeded an unstated threshold for people 65 or older. Gross income is $1,828.67 over the limit, but the engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It treated the full $23,853 as countable and said it exceeded an unstated threshold for people 65 or older. Gross income is $1,828.67 over the limit, but the engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. It chose the aged group correctly, and its decisive error was counting gross income with no disregard, which this corpus records as taxable_income_or_deductions." + }, + { + "model": "gemini-3.7-flash", + "verdict": "ok", + "problems_found": [], + "evidence": "\"Head's income exceeds the non-MAGI Medi-Cal income limits for an elderly individual.\" (no income figure, exclusion or limit anywhere in raw_response)", + "checked_text": "It placed the head in what it called the \"non-MAGI\" elderly pathway but gave no income figure or limit, and it judged income too high, which holds only if California's disregard is skipped. In the engine's senior-or-disabled pathway, $23,853.47 less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It placed the head in what it called the \"non-MAGI\" elderly pathway but gave no income figure or limit, and it judged income too high, which holds against the engine's limit only if California's $230 disregard is skipped or replaced by a smaller one, such as SSI's $20. In the engine's senior-or-disabled pathway, $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. The pathway was right, so the 0 comes from counting income without California's disregard, an income-counting error." + }, + { + "model": "gemini-3.8-flash", + "verdict": "fixed", + "problems_found": [ + "The draft says gross income is \"about 149% of the guideline\" without naming the year. 149% holds only for the 2026 guideline ($23,853.47 / $15,960). Against the 2025 guideline it is about 152%, so the year is needed.", + "The draft only implies the decisive error, testing total income with no disregard. The fix states it outright: the model said \"total income\", which supports it." + ], + "evidence": "\"Head has total income well exceeding California Medi-Cal eligibility limits (138% FPL) for an individual, so Head is not eligible\" (no dollar figures; the 138% FPL threshold is correct)", + "checked_text": "It tested total income against a 138% FPL limit with no disregard and called it \"well\" over, but $23,853.47 is about 149% of the 2026 guideline, only $1,828.67 over the $22,024.80 limit. The engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the limit.", + "final_text": "It tested total income against a 138% FPL limit with no disregard and called it \"well\" over, but $23,853.47 is about 149% of the 2026 guideline, only $1,828.67 over the $22,024.80 limit. The engine's senior-or-disabled pathway subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the limit.", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. Its 138% threshold was right, and its decisive error was testing gross income without California's disregard, an income-counting error." + }, + { + "model": "glm-5.2", + "verdict": "ok", + "problems_found": [], + "evidence": "\"Under PolicyEngine rules, individuals aged 65 and older are generally assumed to be eligible for Medicare, which precludes standalone Medicaid eligibility. The head is 67 years old, so they are not eligible for Medicaid.\" (no income or asset figures in the Medicaid output; the $2,000 resource limit appears only in its SSI chunk)", + "checked_text": "It assumed that Medicare eligibility at 65 or older bars Medicaid and ran no income test. The engine has no Medicare exclusion, and age 67 alone meets the aged test for its senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It assumed that Medicare eligibility at 65 or older bars Medicaid and ran no income test. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group, and age 67 alone meets the aged test for the senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "categorical_eligibility", + "subtype_note": "Kept. Its decisive error was a false categorical bar: it treated Medicare eligibility as ruling out Medicaid, so it never reached the income test." + }, + { + "model": "glm-5.3", + "verdict": "fixed", + "problems_found": [ + "\"Age 67 alone qualifies for the engine's senior-or-disabled pathway\" overstates the rule. Age meets only the aged test; the pathway also has income and asset tests. Changed to \"meets the aged test for\".", + "The draft says \"about 149% of the guideline\" without the year. The figure is right only for the 2026 guideline ($1,987.79 / $1,330); against 2025 it is about 152%.", + "The draft ran to about 78 words, over the style range, so it was tightened to two sentences." + ], + "evidence": "\"Household income of ~$1,988/month (~199% FPL) exceeds California's Medicaid expansion limit of 138% FPL (~$1,600/month) for a single adult; at 67 the head is also beyond CHIP scope.\" Its SNAP output gives the same gross figure: \"~$23,853/12 = ~$1,988, counting Social Security, pension, and IRA distributions\"", + "checked_text": "It applied the adult expansion limit, which excludes anyone over 64, comparing gross income of about $1,988 a month (about 149% of the 2026 guideline, not the 199% it stated) with an understated limit of about $1,600. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $1,987.79 less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline).", + "final_text": "It applied the adult expansion limit, which excludes anyone over 64, comparing gross income of about $1,988 a month (about 149% of the 2026 guideline, not the 199% it stated) with an understated limit of about $1,600. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $1,987.79 less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $1,757.79, under the $1,835.40 limit (138% of the 2026 guideline).", + "subtype": "categorical_eligibility", + "subtype_note": "Kept. Its decisive error was testing the head under the adult expansion group, which excludes him by age, instead of the senior-or-disabled pathway. Its ratio and limit mistakes are secondary." + }, + { + "model": "gpt-5.4-nano", + "verdict": "fixed", + "problems_found": [ + "\"without computing either test\" is unclear: it does not say which two tests are meant. The response gives no Medicaid income figure or limit, so the fix says it compared no income figure with a limit." + ], + "evidence": "\"With substantial retirement income (Social Security plus taxable pension/IRA distributions) and no other disability/low-income qualifying flags provided, Head is estimated not eligible for Medicaid under PolicyEngine rules here.\" (no figures, limit or pathway in the Medicaid output)", + "checked_text": "It denied eligibility because no disability or low-income flag was listed and income was \"substantial\", without comparing any income figure with a limit. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It denied eligibility because no disability or low-income flag was listed and income was \"substantial\", without comparing any income figure with a limit. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway. There, $23,853.47 of income less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion, gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "categorical_eligibility", + "subtype_note": "Kept. Its stated reason was a missing disability or low-income qualifying flag, so it missed that age alone meets the aged test. Its income judgment has no numbers behind it." + }, + { + "model": "gpt-5.5", + "verdict": "fixed", + "problems_found": [ + "\"The engine needs no spend-down\" is not on the fact sheet. Whether the engine models a spend-down at all was not checked, so the sentence was removed.", + "\"reasoning that minimal medical expenses could not bring it down\" goes slightly beyond the text. It was replaced with a closer paraphrase of \"only minimal listed medical expenses\", and the model's own word \"spend-down\" is now in quotation marks." + ], + "evidence": "\"The Head is age 67, so ACA adult Medicaid does not apply; with annual retirement, IRA, and pension income of $23,853 and only minimal listed medical expenses, the Head is above the relevant California aged Medi-Cal income/spend-down standard\" (no threshold figure; the response has only this one chunk)", + "checked_text": "It correctly ruled out the adult expansion group at age 67 but tested the full $23,853 against an unstated aged income or \"spend-down\" standard, with only minimal medical expenses to offset it. The engine's senior-or-disabled pathway instead subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It correctly ruled out the adult expansion group at age 67 but tested the full $23,853 against an unstated aged income or \"spend-down\" standard, with only minimal medical expenses to offset it. The engine's senior-or-disabled pathway instead subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion. That leaves $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. It ruled out the adult group correctly, and its decisive error was counting gross income without California's disregard, an income-counting error." + }, + { + "model": "gpt-5.6-luna", + "verdict": "fixed", + "problems_found": [ + "\"senior-or-disabled pathway, which age 67 alone meets\" overstates the rule. Age meets only the aged test; the pathway also needs the income and asset tests. Changed to \"Age 67 alone meets the aged test for\".", + "Repeated \"California's\" three times; reworded." + ], + "evidence": "\"The head's countable annual income is above the applicable California Medicaid income threshold for this case; not eligible\" (no figures, pathway or threshold anywhere in raw_response)", + "checked_text": "It gave no figures, pathway or threshold and only asserted that countable income was above the applicable California limit, which holds only if the state's $230 monthly disregard is skipped. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $23,853.47 less that disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It gave no figures, pathway or threshold and only asserted that countable income was above the applicable California limit, which holds against the engine's limit only if the state's $230 monthly disregard ($2,760 a year) is skipped or replaced by a smaller one, such as SSI's $20. Age 67 alone meets the aged test for the engine's senior-or-disabled pathway, where $23,853.47 less that disregard (which takes the place of SSI's $20 exclusion) gives $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. It said outright that countable income was over the limit, so the error lies in counting income." + }, + { + "model": "gpt-5.6-sol", + "verdict": "fixed", + "problems_found": [ + "\"That holds only for gross income\" is wrong. Income after only SSI's $20 exclusion ($23,613.47) is still over the limit, so the conclusion holds whenever California's disregard is skipped. The model's SSI output mentions \"the general income exclusion\", so this case is not hypothetical.", + "\"The engine subtracts\" did not name the pathway. Changed to the senior-or-disabled pathway and merged into three sentences." + ], + "evidence": "\"The Head's annual retirement income exceeds California's income limit for the applicable aged Medicaid pathway, so the Head is not eligible\" (no figures). Its SSI output: \"Countable monthly Social Security, pension, and IRA income exceeds the federal SSI payment standard even after the general income exclusion\"", + "checked_text": "It identified the aged pathway but concluded, without showing figures, that retirement income exceeded California's limit. That holds only if California's $230 monthly disregard is skipped: gross income of $23,853.47 is $1,828.67 over the limit. The engine's senior-or-disabled pathway subtracts that disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It identified the aged pathway but concluded, without showing figures, that retirement income exceeded California's limit. That holds against the engine's limit only if California's $230 monthly disregard is skipped or replaced by a smaller one, such as SSI's $20: gross income of $23,853.47 is $1,828.67 over the limit. The engine's senior-or-disabled pathway subtracts that disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Kept. The pathway was right, and its decisive error was counting income without California's disregard." + }, + { + "model": "gpt-5.6-terra", + "verdict": "fixed", + "problems_found": [ + "Minor clarity: the draft wrote '$23,853.47 less California's $230 monthly disregard' as the source of $21,093.47, which mixes an annual figure with a monthly one and reads like a wrong subtraction. Added '($2,760 a year)'. Every other claim checks out: the model's words, the 138% limit of the 2026 guideline ($22,024.80), the $230 disregard replacing the $20 exclusion, and $23,853.47 - $2,760 = $21,093.47.", + "The old Part B mechanism is fully gone. The annotation does not fault the model for choosing the aged pathway, which was correct." + ], + "evidence": "Medicaid explanation: \"At age 67, the head is evaluated under California's aged Medicaid pathway, and the listed retirement income is above the applicable SSI-linked income level; therefore not eligible. value = 0\". It gives no income figure, exclusion or threshold amount and does not mention assets.", + "checked_text": "It placed the head in the aged pathway but tested income against an \"SSI-linked income level,\" giving no figures. The engine's limit for this pathway in California is $22,024.80 (138% of the 2026 poverty guideline), and countable income is $21,093.47: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion.", + "final_text": "It placed the head in the aged pathway but tested income against an \"SSI-linked income level,\" giving no figures. The engine's limit for this pathway in California is $22,024.80 (138% of the 2026 poverty guideline), and countable income is $21,093.47: $23,853.47 less California's $230 monthly disregard ($2,760 a year), which takes the place of SSI's $20 exclusion.", + "subtype": "thresholds_rates", + "subtype_note": "The only error the response names is the threshold: an SSI-linked income level instead of the pathway's 138% FPL limit. No income or exclusion figure is given to fault, so the current thresholds_rates is kept." + }, + { + "model": "gpt-6-astra", + "verdict": "ok", + "problems_found": [ + "No errors. The model statements are supported: it ruled out adult expansion at 67, and it said retirement income after the general income exclusion exceeded California's aged limit, with no figures. The engine claims match the facts: the $230 disregard is passed in as the general exclusion, $23,853.47 - $2,760 = $21,093.47, under $22,024.80. No Part B or premium language remains, and the annotation does not fault the correct adult-expansion ruling." + ], + "evidence": "Medicaid explanation: \"at age 67 the adult expansion pathway does not apply, and retirement income after the general income exclusion exceeds California's aged Medicaid income limit. value = 0\". It states no amount for the exclusion, the income or the limit.", + "checked_text": "It correctly ruled out the adult expansion group at 67 but said retirement income after the general income exclusion exceeded California's aged limit, giving no figures. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "final_text": "It correctly ruled out the adult expansion group at 67 but said retirement income after the general income exclusion exceeded California's aged limit, giving no figures. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "The stated reasoning is a countable-income test after an exclusion, and the engine's result depends on how large that exclusion is. Corpus practice classes this as taxable_income_or_deductions, so it is kept." + }, + { + "model": "gpt-6-sol", + "verdict": "ok", + "problems_found": [ + "No errors. The model said its retirement income exceeded California's aged threshold and named no exclusion or figure, which the draft states accurately. The engine figures check out: $23,853.47 gross is over $22,024.80, and $23,853.47 - $2,760 = $21,093.47 is under it. The disregard is correctly described as replacing the $20 exclusion. No Part B language remains." + ], + "evidence": "Medicaid explanation: \"The head's retirement income exceeds the California income threshold for aged Medicaid eligibility and the head is not eligible; value = 0\". It names no exclusion, income figure or threshold amount.", + "checked_text": "It said the head's retirement income exceeded California's aged Medicaid threshold, naming no exclusion and giving no figures. Gross income of $23,853.47 is over the $22,024.80 limit (138% of the 2026 poverty guideline), but the engine first subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving countable income of $21,093.47, under the limit.", + "final_text": "It said the head's retirement income exceeded California's aged Medicaid threshold, naming no exclusion and giving no figures. Gross income of $23,853.47 is over the $22,024.80 limit (138% of the 2026 poverty guideline), but the engine first subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving countable income of $21,093.47, under the limit.", + "subtype": "taxable_income_or_deductions", + "subtype_note": "Gross income is over the limit and countable income is under it, so the case turns on the income disregard. That is a countable-income error, which the corpus classes as taxable_income_or_deductions. The current subtype is kept." + }, + { + "model": "gpt-6.1-sol", + "verdict": "fixed", + "problems_found": [ + "The draft said the model gave its answer 'without stating the exclusion', but the model did name it ('the applicable general income exclusion'). What it left out was the amount. Changed to 'without stating its amount, the income or the limit'.", + "Everything else checks out: the $230 disregard is passed in as the general exclusion and replaces the $20, $23,853.47 - $2,760 = $21,093.47, and the limit is $22,024.80. No Part B or premium language remains." + ], + "evidence": "Medicaid explanation: \"retirement income exceeds California's aged-person Medi-Cal income limit after the applicable general income exclusion, and no other qualifying pathway is listed; value = 0\". It states no amount for the exclusion, the income or the limit.", + "checked_text": "It said retirement income exceeded California's aged limit after \"the applicable general income exclusion,\" without stating its amount, the income or the limit. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "final_text": "It said retirement income exceeded California's aged limit after \"the applicable general income exclusion,\" without stating its amount, the income or the limit. In the engine that exclusion is California's $230 monthly disregard, which takes the place of SSI's $20, so countable income is $21,093.47 ($23,853.47 less $2,760), under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "The stated reasoning is a countable-income test after a general exclusion, and the engine's answer depends on how large that exclusion is. Corpus practice uses taxable_income_or_deductions, so it is kept." + }, + { + "model": "grok-4.3", + "verdict": "fixed", + "problems_found": [ + "The draft dropped the current annotation's accurate point that the head also passes the asset test. Added that $4,200 is under the $130,000 limit. This model never reached any income or asset test, so the full path to eligibility should be stated.", + "The draft wrote '$23,853.47 less California's $230 monthly disregard', which mixes an annual figure with a monthly one. Added '$2,760 a year'.", + "The rest is correct. 'full dual eligibles' appears only as the model's own words. The engine has no Medicare exclusion, and age 67 meets the aged test. The Part B mechanism is gone." + ], + "evidence": "Medicaid explanation: \"Head is Medicare-eligible; Medicaid eligibility rules exclude full dual eligibles under PolicyEngine. value = 0\". The content says only \"The tool call follows.\" There is no Medicaid income or asset analysis.", + "checked_text": "It ruled the head out as Medicare-eligible, claiming the engine excludes \"full dual eligibles\" from Medicaid. The engine has no Medicare exclusion: at 67 the head counts as aged for the senior-or-disabled pathway, countable income of $21,093.47 ($23,853.47 less California's $230 monthly disregard, $2,760 a year) is under the $22,024.80 limit (138% of the 2026 poverty guideline), and $4,200 in assets is under the $130,000 limit.", + "final_text": "It ruled the head out as Medicare-eligible, claiming the engine excludes \"full dual eligibles\" from Medicaid. In the engine's Medicaid rules, Medicare eligibility bars only the adult expansion group: at 67 the head counts as aged for the senior-or-disabled pathway, countable income of $21,093.47 ($23,853.47 less California's $230 monthly disregard, $2,760 a year) is under the $22,024.80 limit (138% of the 2026 poverty guideline), and $4,200 in assets is under the $130,000 limit.", + "subtype": "categorical_eligibility", + "subtype_note": "The deciding error is a false categorical exclusion: it treated Medicare eligibility as a bar to Medicaid and never ran an income or asset test. categorical_eligibility is correct and is kept." + }, + { + "model": "grok-4.5", + "verdict": "ok", + "problems_found": [ + "No errors. The model statements are supported: it put about $23,853 of countable income (the gross total, with no disregard) and $4,200 in bank assets over California's aged limits. The engine figures check out: the asset limit is $130,000, $23,853.47 - $2,760 = $21,093.47, and the limit is $22,024.80. No Part B language remains, and the annotation does not fault the correct aged pathway." + ], + "evidence": "Medicaid explanation: \"Age 67 ABD pathway; countable income ~$23,853 and bank assets $4,200 both exceed CA Medi-Cal aged resource and income limits for full Medicaid or Medicare Savings Programs. value = 0\".", + "checked_text": "It said bank assets of $4,200 and countable income of about $23,853 both exceeded California's aged limits, counting gross income in full. The pathway's asset limit is $130,000, so $4,200 passes, and the engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "final_text": "It said bank assets of $4,200 and countable income of about $23,853 both exceeded California's aged limits, counting gross income in full. The pathway's asset limit is $130,000, so $4,200 passes, and the engine subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "subtype": "asset_resource", + "subtype_note": "It made two independent errors, and either one alone produces 0: it treated $4,200 as over the resource limit, and it counted gross income with no disregard. The explicit asset error makes the current asset_resource defensible, so it is kept." + }, + { + "model": "grok-4.6", + "verdict": "ok", + "problems_found": [ + "No errors. The quotes match the response exactly: 'MAGI/ABD countable income' of $23,853 and 'Aged & Disabled FPL'. The real-world program name appears only as the model's own words. The engine claims match the facts: income is counted under SSI rules, the $230 disregard replaces the $20 exclusion, the result is $21,093.47, and the limit is $22,024.80. No Part B language remains." + ], + "evidence": "Medicaid explanation: \"Head is 67 with Medicare-age status and MAGI/ABD countable income of $23,853, which exceeds California's Aged & Disabled FPL and Medicare Savings Program limits, so not eligible. value = 0\". It names no exclusion or limit amount.", + "checked_text": "It counted the full $23,853 as \"MAGI/ABD countable income\" and said it exceeded California's \"Aged & Disabled FPL\" limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "final_text": "It counted the full $23,853 as \"MAGI/ABD countable income\" and said it exceeded California's \"Aged & Disabled FPL\" limit. The engine counts that income under SSI rules but subtracts California's $230 monthly disregard, which takes the place of SSI's $20 exclusion, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 poverty guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "The deciding error is counting gross income with no disregard, a countable-income error. The corpus classes these as taxable_income_or_deductions, so it is kept." + }, + { + "model": "grok-4.7", + "verdict": "ok", + "problems_found": [ + "No errors. The model explicitly used only the $20 monthly disregard and the 138% FPL limit, and the annotation does not fault the threshold, which was correct. The arithmetic checks out: $23,853.47 - $240 = $23,613.47, over $22,024.80, and $23,853.47 - $2,760 = $21,093.47, under it. The $230 disregard is correctly described as replacing the $20. Monthly figures are appropriate here because the model worked in monthly terms. No Part B language remains." + ], + "evidence": "Medicaid explanation: \"After the $20 monthly unearned-income disregard, countable income exceeds California's 138% FPL Aged and Disabled Medi-Cal limit, so the head is not eligible for Medicaid. value = 0\".", + "checked_text": "It applied only the $20 monthly unearned-income disregard before testing against California's 138% FPL limit, which leaves about $23,613, still over the $22,024.80 limit. The engine uses California's $230 monthly disregard in place of that $20 exclusion, so countable income is $21,093.47 ($23,853.47 less $2,760), under the limit.", + "final_text": "It applied only the $20 monthly unearned-income disregard before testing against California's 138% FPL limit, which leaves about $23,613, still over the $22,024.80 limit. The engine uses California's $230 monthly disregard in place of that $20 exclusion, so countable income is $21,093.47 ($23,853.47 less $2,760), under the limit.", + "subtype": "taxable_income_or_deductions", + "subtype_note": "The threshold was right, and the only error is the size of the income disregard ($20 instead of California's $230). That is a countable-income error, taxable_income_or_deductions in corpus practice, so it is kept." + }, + { + "model": "grok-build-0.1", + "verdict": "ok", + "problems_found": [], + "evidence": "Medicaid explanation: \"Bank account assets of $4,200 exceed the $2,000 asset limit for Medi-Cal eligibility for aged individuals in California.\" SSI answer: \"exceed the $2,000 SSI resource limit\". The Medicaid answer has no income test.", + "checked_text": "It applied a $2,000 asset limit to aged Medi-Cal and denied eligibility because the $4,200 in bank assets exceeded it, but $2,000 is SSI's resource limit, not this pathway's test. The engine's California senior-or-disabled pathway allows assets below $130,000, and countable income is $23,853.47 less California's $230 monthly disregard, or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It applied a $2,000 asset limit to aged Medicaid and denied eligibility because the $4,200 in bank assets exceeded it, but $2,000 is SSI's resource limit, not this pathway's test. The engine's California senior-or-disabled pathway allows assets below $130,000, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "asset_resource", + "subtype_note": "The model's only reason for 0 was the $2,000 asset limit, so asset_resource is correct. Kept." + }, + { + "model": "inkling", + "verdict": "ok", + "problems_found": [], + "evidence": "\"CA Medi-Cal for aged individuals generally requires income at or below 100% FPL (~$15,650/year); the head's ~$23,853 annual income exceeds that limit\"", + "checked_text": "It applied a 100% FPL limit of about $15,650, the 2025 guideline, and compared gross income of about $23,853 against it. The engine uses 138% of the 2026 guideline, $22,024.80, and subtracts California's $230 monthly disregard (in place of SSI's $20 exclusion) from $23,853.47, leaving countable income of $21,093.47, under the limit.", + "final_text": "It applied a 100% FPL limit of about $15,650, the 2025 guideline, and compared gross income of about $23,853 against it. The engine uses 138% of the 2026 guideline, $22,024.80, and subtracts California's $230 monthly disregard ($2,760 a year, in place of SSI's $20 exclusion) from $23,853.47, leaving countable income of $21,093.47, under the limit.", + "subtype": "thresholds_rates", + "subtype_note": "The 100% FPL threshold is decisive on its own: even the correct $21,093.47 fails $15,650. Kept." + }, + { + "model": "kimi-k2.6", + "verdict": "fixed", + "problems_found": [ + "The draft says \"age 67 meets it\" about the pathway. That repeats the overclaim the draft itself flagged in the old annotation: age meets only the aged criterion, and the pathway also needs the income and asset tests. Changed to \"age 67 alone meets its aged criterion\".", + "\"needs neither disability nor SSI receipt\" left out blindness, which the model also cited. Reworded to \"needs no disability, blindness or SSI receipt\"." + ], + "evidence": "\"California Medicaid expansion covers adults ages 19 through 64, so the Head at age 67 is outside that age group. The Head is also not indicated as disabled, blind, pregnant, or an SSI recipient, and therefore does not qualify for categorical non-MAGI aged or disabled Medicaid pathways\"", + "checked_text": "It correctly placed the head outside the adult expansion group but ruled out the aged and disabled pathways because the head was not disabled, blind or an SSI recipient. The engine's senior-or-disabled pathway needs no disability, blindness or SSI receipt; age 67 alone meets its aged criterion. Countable income is $23,853.47 less California's $230 monthly disregard, or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It correctly placed the head outside the adult expansion group but ruled out the aged and disabled pathways because the head was not disabled, blind or an SSI recipient. The engine's senior-or-disabled pathway needs no disability, blindness or SSI receipt; age 67 alone meets its aged criterion. Countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "categorical_eligibility", + "subtype_note": "The only reason given was category: no income or asset test was run. Kept." + }, + { + "model": "kimi-k3", + "verdict": "ok", + "problems_found": [], + "evidence": "\"At 67 Head is outside the ACA adult expansion group, and the listed retirement income is above PolicyEngine's California aged Medi-Cal income limit, so Head is not Medicaid eligible.\" It names no limit, gives no figure and applies no disregard.", + "checked_text": "It correctly placed the head outside the adult expansion group but said the listed retirement income exceeded the California aged income limit, without naming a limit or applying any disregard. The engine subtracts California's $230 monthly disregard from the $23,853.47 of Social Security, pension and IRA income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It correctly placed the head outside the adult expansion group but said the listed retirement income exceeded the California aged income limit, without naming a limit or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "A countable-income error with unadjusted income and no disregard. The corpus files these under taxable_income_or_deductions. Kept." + }, + { + "model": "minimax-m3", + "verdict": "ok", + "problems_found": [], + "evidence": "\"Income ~$24k/year for single. CA Medicaid for 65+: income limit higher. However, head_medicaid_eligible under PolicyEngine may differ. With SS $9,445 and pension $13,608, countable income may exceed limit. Likely 0.\"", + "checked_text": "It guessed that about $24,000 of income might exceed an unstated senior limit and answered 0 without naming a threshold or applying any disregard. The engine subtracts California's $230 monthly disregard from the $23,853.47 of Social Security, pension and IRA income, leaving countable income of $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It guessed that about $24,000 of income might exceed an unstated senior limit and answered 0 without naming a threshold or applying any disregard. The engine subtracts California's $230 monthly disregard ($2,760 a year) from the $23,853.47 of Social Security, pension and IRA income, leaving countable income of $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "It used unadjusted income with no disregard, a countable-income error. Kept." + }, + { + "model": "ox-alpha", + "verdict": "ok", + "problems_found": [], + "evidence": "\"At 65+ the head falls in the aged category, where Medi-Cal/Medicaid eligibility follows SSI-related rules; he fails the $2,000 resource limit ($4,200 in bank assets) and his ~$24,000 income also exceeds aged-program income standards\"", + "checked_text": "It correctly treated the head as aged but applied SSI's $2,000 resource limit to the $4,200 in bank assets and said gross income of about $24,000 exceeded the aged income standard. The engine's California pathway allows assets below $130,000 and subtracts a $230 monthly disregard from the $23,853.47 of income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It correctly treated the head as aged but applied SSI's $2,000 resource limit to the $4,200 in bank assets and said gross income of about $24,000 exceeded the aged income standard. The engine's California pathway allows assets below $130,000 and subtracts a $230 monthly disregard ($2,760 a year) from the $23,853.47 of income, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "asset_resource", + "subtype_note": "Both the asset error and the income error are decisive. The model named the asset test first, so asset_resource is not clearly wrong. Kept." + }, + { + "model": "qwen-3.7-max", + "verdict": "fixed", + "problems_found": [ + "\"since age 67 meets it\" overclaims: age meets only the pathway's aged criterion, and the income and asset tests still apply. Changed to \"age 67 alone meets its aged criterion\".", + "The draft does not say what the $230 disregard is subtracted from. Added \"from $23,853.47\" so $21,093.47 follows from it.", + "The paraphrase \"aged, blind and disabled limits\" names a real-world program category without quoting the model. Replaced with the model's exact words in quotes." + ], + "evidence": "\"Head is 67 and thus qualifies for Medicare, not Medicaid through the ACA MAGI pathways. Head's countable income (Social Security $9,445 + taxable IRA $800 + taxable pension $13,608 = $23,853) exceeds typical aged/blind/disabled Medicaid limits, and no disability, nursing home, or SSI status is indicated.\"", + "checked_text": "It counted $23,853 of Social Security, pension and IRA income in full, said it exceeded \"typical aged/blind/disabled Medicaid limits\" without naming one, and cited the missing disability or SSI status. The engine's senior-or-disabled pathway needs neither, since age 67 alone meets its aged criterion, and it subtracts California's $230 monthly disregard from $23,853.47, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It counted $23,853 of Social Security, pension and IRA income in full, said it exceeded \"typical aged/blind/disabled Medicaid limits\" without naming one, and cited the missing disability or SSI status. The engine's senior-or-disabled pathway needs neither, since age 67 alone meets its aged criterion, and it subtracts California's $230 monthly disregard ($2,760 a year) from $23,853.47, leaving $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "taxable_income_or_deductions", + "subtype_note": "The main stated reason was income counted in full with no disregard, and missing status was secondary. The current subtype is not clearly wrong. Kept." + }, + { + "model": "qwen3.8-max", + "verdict": "fixed", + "problems_found": [ + "\"Age 65 or older alone meets the engine's senior-or-disabled pathway\" overclaims: age meets only the aged criterion, and the pathway also needs the income and asset tests. Changed to \"meets the aged criterion of the engine's senior-or-disabled pathway\"." + ], + "evidence": "\"The 67-year-old Head is not disabled, blind, pregnant, or otherwise in an eligibility category listed, and their income is above the applicable Medicaid pathway for a senior in this household\". The Medicaid answer gives no income figure or limit.", + "checked_text": "It said the 67-year-old was not disabled, blind or in any other listed category, and that income was above the senior pathway's limit without naming a figure. Age 65 or older alone meets the aged criterion of the engine's senior-or-disabled pathway, and countable income is $23,853.47 less California's $230 monthly disregard, or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "final_text": "It said the 67-year-old was not disabled, blind or in any other listed category, and that income was above the senior pathway's limit without naming a figure. Age 65 or older alone meets the aged criterion of the engine's senior-or-disabled pathway, and countable income is $23,853.47 less California's $230 monthly disregard ($2,760 a year), or $21,093.47, under the $22,024.80 limit (138% of the 2026 guideline).", + "subtype": "categorical_eligibility", + "subtype_note": "The lead reason was that the head fit no category, and the income claim gave no figure. Kept." + } + ] +} diff --git a/tests/test_annotation_rewrites.py b/tests/test_annotation_rewrites.py new file mode 100644 index 00000000..519e6d89 --- /dev/null +++ b/tests/test_annotation_rewrites.py @@ -0,0 +1,135 @@ +"""Committed rewrites of frozen audit annotations stay in force and stay wording-only. + +reference_audit/2026-10-05-medicaid-031-annotations/rewrites.json corrects the text +that explained scenario_031's head Medicaid reference with a Medicare Part B premium +the engine never subtracts. A later release that rebuilds the annotation files from +an older base must carry these rewrites (or supersede the ledger), or these tests +fail. +""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +AUDIT = ROOT / "reference_audit" / "2026-10-05-medicaid-031-annotations" +MANIFEST = ROOT / "paper" / "snapshot" / "20260501" / "manifest.json" +KEYS = { + "us_audit_row_annotations.csv": ("scenario_id", "variable", "model"), + "us_case_notes.csv": ("scenario_id", "variable"), + "us_case_reference_explanations.csv": ("scenario_id", "variable"), +} +FIELDS = { + "us_audit_row_annotations.csv": "annotation", + "us_case_notes.csv": "case_annotation", + "us_case_reference_explanations.csv": "explanation", +} + + +def _rewrites() -> list[dict]: + return json.loads((AUDIT / "rewrites.json").read_text())["rewrites"] + + +def _annotation_dir() -> Path: + manifest = json.loads(MANIFEST.read_text()) + return ROOT / manifest["audit_annotation_artifacts"]["path"] + + +def _rows(name: str) -> list[dict]: + with (_annotation_dir() / name).open(newline="") as source: + return list(csv.DictReader(source)) + + +def test_every_rewrite_is_in_force(): + for name, key_columns in KEYS.items(): + items = [item for item in _rewrites() if item["file"] == name] + rows = _rows(name) + index = {} + for row in rows: + index.setdefault(tuple(row[c] for c in key_columns), []).append(row) + for item in items: + key = tuple(item[c] for c in key_columns) + assert len(index.get(key, [])) == 1, (name, key) + assert index[key][0][FIELDS[name]] == item["new"], (name, key) + + +def test_no_rewritten_text_survives_anywhere(): + old_texts = {item["old"] for item in _rewrites()} + for name, field in FIELDS.items(): + for row in _rows(name): + assert row[field] not in old_texts, (name, row["scenario_id"]) + + +def test_rewrites_are_wording_only_and_well_formed(): + rewrites = _rewrites() + assert rewrites + keys = set() + for item in rewrites: + assert item["file"] in KEYS + assert item["field"] == FIELDS[item["file"]] + assert item["old"] != item["new"] + assert item["reason"].strip() + key = (item["file"], *(item[c] for c in KEYS[item["file"]])) + assert key not in keys, key + keys.add(key) + # A rewrite carries text only: no class, source or subtype field. + assert set(item) <= { + "file", + "field", + "scenario_id", + "variable", + "model", + "old", + "new", + "reason", + } + + +def test_rewritten_rows_keep_their_classes(): + """The rows stay model errors; the rewrites change no failure class.""" + rewritten = { + (item["scenario_id"], item["variable"], item["model"]) + for item in _rewrites() + if item["file"] == "us_audit_row_annotations.csv" + } + rows = [ + row + for row in _rows("us_audit_row_annotations.csv") + if (row["scenario_id"], row["variable"], row["model"]) in rewritten + ] + assert len(rows) == len(rewritten) + assert {row["failure_source"] for row in rows} == {"llm_error"} + assert {row["reference_suspect"] for row in rows} == {"False"} + + +def test_engine_record_backs_the_figures_the_text_states(): + """The recomputed engine values are the ones the case note and explanation give.""" + record = json.loads((AUDIT / "verification" / "engine_values.json").read_text()) + summary = record["summary"] + assert record["published_reference"] == 1.0 + assert summary["countable_income"] <= summary["income_limit"] + for reading in ("no_part_b", "not_enrolled"): + assert ( + record["readings"][reading][ + "medicaid_optional_senior_or_disabled_countable_income" + ] + == record["readings"]["modeled"][ + "medicaid_optional_senior_or_disabled_countable_income" + ] + ) + figures = [ + f"${summary['gross_unearned_income']:,.2f}", + f"${summary['countable_income']:,.2f}", + f"${summary['income_limit']:,.2f}", + ] + for name in ("us_case_notes.csv", "us_case_reference_explanations.csv"): + (row,) = [ + row + for row in _rows(name) + if (row["scenario_id"], row["variable"]) + == ("scenario_031", "head_medicaid_eligible") + ] + for figure in figures: + assert figure in row[FIELDS[name]], (name, figure)